2024-12-04 08:23:29,271 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-12-04 08:23:29,283 main DEBUG Took 0.009604 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 08:23:29,283 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 08:23:29,283 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 08:23:29,284 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 08:23:29,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,291 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 08:23:29,303 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,304 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,305 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,306 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,307 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,308 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,309 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,309 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,310 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,311 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,311 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,312 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 08:23:29,313 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,313 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 08:23:29,314 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 08:23:29,316 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 08:23:29,317 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 08:23:29,318 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 08:23:29,319 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 08:23:29,319 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 08:23:29,327 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 08:23:29,330 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 08:23:29,331 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 08:23:29,332 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 08:23:29,332 main DEBUG createAppenders(={Console}) 2024-12-04 08:23:29,333 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-12-04 08:23:29,333 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-12-04 08:23:29,333 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-12-04 08:23:29,334 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 08:23:29,334 main DEBUG OutputStream closed 2024-12-04 08:23:29,334 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 08:23:29,335 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 08:23:29,335 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-12-04 08:23:29,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 08:23:29,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 08:23:29,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 08:23:29,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 08:23:29,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 08:23:29,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 08:23:29,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 08:23:29,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 08:23:29,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 08:23:29,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 08:23:29,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 08:23:29,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 08:23:29,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 08:23:29,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 08:23:29,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 08:23:29,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 08:23:29,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 08:23:29,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 08:23:29,412 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 08:23:29,412 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-12-04 08:23:29,412 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 08:23:29,413 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-12-04T08:23:29,633 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d 2024-12-04 08:23:29,636 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 08:23:29,636 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T08:23:29,645 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-12-04T08:23:29,681 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=245, ProcessCount=11, AvailableMemoryMB=7167 2024-12-04T08:23:29,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:23:29,705 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3, deleteOnExit=true 2024-12-04T08:23:29,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:23:29,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/test.cache.data in system properties and HBase conf 2024-12-04T08:23:29,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:23:29,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:23:29,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:23:29,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:23:29,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:23:29,807 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T08:23:29,904 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:23:29,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:23:29,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:23:29,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:23:29,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:23:29,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:23:29,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:23:29,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:23:29,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:23:29,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:23:29,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:23:29,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:23:29,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:23:29,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:23:29,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:23:30,427 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:23:30,762 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T08:23:30,838 INFO [Time-limited test {}] log.Log(170): Logging initialized @2250ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T08:23:30,914 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:23:30,975 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:23:30,994 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:23:30,995 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:23:30,996 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:23:31,008 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:23:31,010 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:23:31,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:23:31,202 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/java.io.tmpdir/jetty-localhost-43643-hadoop-hdfs-3_4_1-tests_jar-_-any-13767366239093794512/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:23:31,211 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:43643} 2024-12-04T08:23:31,212 INFO [Time-limited test {}] server.Server(415): Started @2625ms 2024-12-04T08:23:31,253 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:23:31,602 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:23:31,608 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:23:31,609 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:23:31,609 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:23:31,609 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:23:31,610 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:23:31,611 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:23:31,731 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/java.io.tmpdir/jetty-localhost-38889-hadoop-hdfs-3_4_1-tests_jar-_-any-12279258557865346341/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:23:31,732 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:38889} 2024-12-04T08:23:31,732 INFO [Time-limited test {}] server.Server(415): Started @3146ms 2024-12-04T08:23:31,791 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:23:31,934 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:23:31,941 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:23:31,955 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:23:31,956 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:23:31,956 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:23:31,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:23:31,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:23:32,086 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/java.io.tmpdir/jetty-localhost-39489-hadoop-hdfs-3_4_1-tests_jar-_-any-6215204298719426499/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:23:32,087 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:39489} 2024-12-04T08:23:32,087 INFO [Time-limited test {}] server.Server(415): Started @3501ms 2024-12-04T08:23:32,090 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:23:32,323 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data1/current/BP-864805852-172.17.0.2-1733300610521/current, will proceed with Du for space computation calculation, 2024-12-04T08:23:32,324 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data2/current/BP-864805852-172.17.0.2-1733300610521/current, will proceed with Du for space computation calculation, 2024-12-04T08:23:32,325 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data3/current/BP-864805852-172.17.0.2-1733300610521/current, will proceed with Du for space computation calculation, 2024-12-04T08:23:32,337 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data4/current/BP-864805852-172.17.0.2-1733300610521/current, will proceed with Du for space computation calculation, 2024-12-04T08:23:32,395 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:23:32,395 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:23:32,479 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa049eaa6ed7bfcad with lease ID 0xa03137a615fc3d3d: Processing first storage report for DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1 from datanode DatanodeRegistration(127.0.0.1:38971, datanodeUuid=c3f2aa86-8653-489c-aa20-7708c4d2db01, infoPort=36577, infoSecurePort=0, ipcPort=45497, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521) 2024-12-04T08:23:32,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa049eaa6ed7bfcad with lease ID 0xa03137a615fc3d3d: from storage DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1 node DatanodeRegistration(127.0.0.1:38971, datanodeUuid=c3f2aa86-8653-489c-aa20-7708c4d2db01, infoPort=36577, infoSecurePort=0, ipcPort=45497, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-04T08:23:32,482 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaaa23f1666054c6 with lease ID 0xa03137a615fc3d3e: Processing first storage report for DS-137e4273-5e71-45b3-8a2f-e69da349b8b9 from datanode DatanodeRegistration(127.0.0.1:44543, datanodeUuid=621430fd-59eb-4131-9539-9e1f04696325, infoPort=45747, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521) 2024-12-04T08:23:32,482 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaaa23f1666054c6 with lease ID 0xa03137a615fc3d3e: from storage DS-137e4273-5e71-45b3-8a2f-e69da349b8b9 node DatanodeRegistration(127.0.0.1:44543, datanodeUuid=621430fd-59eb-4131-9539-9e1f04696325, infoPort=45747, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:23:32,482 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaaa23f1666054c6 with lease ID 0xa03137a615fc3d3e: Processing first storage report for DS-dc7732e3-e3cd-47ed-9281-ced12c8d0de0 from datanode DatanodeRegistration(127.0.0.1:44543, datanodeUuid=621430fd-59eb-4131-9539-9e1f04696325, infoPort=45747, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521) 2024-12-04T08:23:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaaa23f1666054c6 with lease ID 0xa03137a615fc3d3e: from storage DS-dc7732e3-e3cd-47ed-9281-ced12c8d0de0 node DatanodeRegistration(127.0.0.1:44543, datanodeUuid=621430fd-59eb-4131-9539-9e1f04696325, infoPort=45747, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T08:23:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa049eaa6ed7bfcad with lease ID 0xa03137a615fc3d3d: Processing first storage report for DS-d8e50748-2e7d-416c-9d66-7c73b11f4008 from datanode DatanodeRegistration(127.0.0.1:38971, datanodeUuid=c3f2aa86-8653-489c-aa20-7708c4d2db01, infoPort=36577, infoSecurePort=0, ipcPort=45497, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521) 2024-12-04T08:23:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa049eaa6ed7bfcad with lease ID 0xa03137a615fc3d3d: from storage DS-d8e50748-2e7d-416c-9d66-7c73b11f4008 node DatanodeRegistration(127.0.0.1:38971, datanodeUuid=c3f2aa86-8653-489c-aa20-7708c4d2db01, infoPort=36577, infoSecurePort=0, ipcPort=45497, storageInfo=lv=-57;cid=testClusterID;nsid=849444122;c=1733300610521), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:23:32,537 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d 2024-12-04T08:23:32,642 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/zookeeper_0, clientPort=51944, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:23:32,655 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51944 2024-12-04T08:23:32,671 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:32,676 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:32,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:23:32,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:23:33,378 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce with version=8 2024-12-04T08:23:33,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:23:33,475 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T08:23:33,723 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:23:33,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:23:33,735 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:23:33,739 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:23:33,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:23:33,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:23:33,875 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:23:33,935 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T08:23:33,944 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T08:23:33,947 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:23:33,975 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7718 (auto-detected) 2024-12-04T08:23:33,976 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T08:23:33,996 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42893 2024-12-04T08:23:34,019 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42893 connecting to ZooKeeper ensemble=127.0.0.1:51944 2024-12-04T08:23:34,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428930x0, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:23:34,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42893-0x1017c9258970000 connected 2024-12-04T08:23:34,095 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:34,099 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:34,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:23:34,116 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce, hbase.cluster.distributed=false 2024-12-04T08:23:34,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:23:34,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42893 2024-12-04T08:23:34,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42893 2024-12-04T08:23:34,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42893 2024-12-04T08:23:34,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42893 2024-12-04T08:23:34,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42893 2024-12-04T08:23:34,268 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:23:34,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:23:34,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:23:34,270 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:23:34,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:23:34,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:23:34,273 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:23:34,275 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:23:34,276 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40167 2024-12-04T08:23:34,278 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40167 connecting to ZooKeeper ensemble=127.0.0.1:51944 2024-12-04T08:23:34,279 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:34,283 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:34,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401670x0, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:23:34,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:401670x0, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:23:34,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40167-0x1017c9258970001 connected 2024-12-04T08:23:34,298 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:23:34,306 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:23:34,309 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:23:34,314 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:23:34,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40167 2024-12-04T08:23:34,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40167 2024-12-04T08:23:34,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40167 2024-12-04T08:23:34,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40167 2024-12-04T08:23:34,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40167 2024-12-04T08:23:34,335 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:42893 2024-12-04T08:23:34,336 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:34,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:23:34,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:23:34,345 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:34,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:23:34,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:34,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:34,368 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:23:34,371 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,42893,1733300613527 from backup master directory 2024-12-04T08:23:34,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:34,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:23:34,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:23:34,374 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:23:34,374 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:34,376 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T08:23:34,378 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T08:23:34,432 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase.id] with ID: 56cf4954-f064-4471-bb83-ad6d0ad05eb9 2024-12-04T08:23:34,433 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/.tmp/hbase.id 2024-12-04T08:23:34,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:23:34,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:23:34,446 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/.tmp/hbase.id]:[hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase.id] 2024-12-04T08:23:34,490 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:34,494 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:23:34,514 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-04T08:23:34,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:34,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:34,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:23:34,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:23:34,551 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:23:34,553 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:23:34,559 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:23:34,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:23:34,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:23:34,610 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store 2024-12-04T08:23:34,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:23:34,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:23:34,634 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T08:23:34,637 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:23:34,638 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:23:34,638 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:23:34,639 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:23:34,640 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:23:34,640 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:23:34,640 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:23:34,642 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300614638Disabling compacts and flushes for region at 1733300614638Disabling writes for close at 1733300614640 (+2 ms)Writing region close event to WAL at 1733300614640Closed at 1733300614640 2024-12-04T08:23:34,644 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/.initializing 2024-12-04T08:23:34,644 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/WALs/f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:34,668 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C42893%2C1733300613527, suffix=, logDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/WALs/f5a5a857f5c5,42893,1733300613527, archiveDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/oldWALs, maxLogs=10 2024-12-04T08:23:34,681 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C42893%2C1733300613527.1733300614675 2024-12-04T08:23:34,704 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/WALs/f5a5a857f5c5,42893,1733300613527/f5a5a857f5c5%2C42893%2C1733300613527.1733300614675 2024-12-04T08:23:34,716 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:23:34,717 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:23:34,718 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:23:34,721 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,722 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,788 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:23:34,792 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:34,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:34,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:23:34,800 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:34,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:23:34,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:23:34,804 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:34,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:23:34,806 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:23:34,809 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:34,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:23:34,810 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,813 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,815 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,819 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,820 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,823 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:23:34,827 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:23:34,831 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:23:34,833 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732534, jitterRate=-0.06853584945201874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:23:34,841 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300614736Initializing all the Stores at 1733300614738 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300614739 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300614740 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300614740Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300614740Cleaning up temporary data from old regions at 1733300614820 (+80 ms)Region opened successfully at 1733300614841 (+21 ms) 2024-12-04T08:23:34,843 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:23:34,884 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1847ffe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:23:34,916 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:23:34,927 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:23:34,928 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:23:34,931 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:23:34,933 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T08:23:34,937 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T08:23:34,938 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:23:34,966 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:23:34,974 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:23:34,977 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:23:34,979 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:23:34,980 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:23:34,984 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:23:34,986 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:23:34,990 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:23:34,991 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:23:34,993 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:23:34,994 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:23:35,012 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:23:35,013 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:23:35,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:23:35,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:23:35,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,021 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,42893,1733300613527, sessionid=0x1017c9258970000, setting cluster-up flag (Was=false) 2024-12-04T08:23:35,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,040 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:23:35,042 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:35,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,059 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:23:35,061 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:35,067 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:23:35,123 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(746): ClusterId : 56cf4954-f064-4471-bb83-ad6d0ad05eb9 2024-12-04T08:23:35,126 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:23:35,131 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:23:35,132 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:23:35,136 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:23:35,137 DEBUG [RS:0;f5a5a857f5c5:40167 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64908e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:23:35,142 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:23:35,152 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:40167 2024-12-04T08:23:35,152 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:23:35,155 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:23:35,155 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:23:35,155 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:23:35,158 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,42893,1733300613527 with port=40167, startcode=1733300614228 2024-12-04T08:23:35,159 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:23:35,165 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,42893,1733300613527 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:23:35,171 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:23:35,172 DEBUG [RS:0;f5a5a857f5c5:40167 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:23:35,172 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:23:35,172 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:23:35,172 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:23:35,172 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:23:35,172 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,172 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:23:35,173 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,177 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300645177 2024-12-04T08:23:35,178 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:23:35,178 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:23:35,179 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:23:35,180 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:23:35,184 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:23:35,185 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:23:35,185 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:23:35,186 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:23:35,186 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,186 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:23:35,187 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,191 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:23:35,192 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:23:35,193 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:23:35,197 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:23:35,197 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:23:35,202 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300615199,5,FailOnTimeoutGroup] 2024-12-04T08:23:35,203 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300615202,5,FailOnTimeoutGroup] 2024-12-04T08:23:35,203 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,203 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:23:35,205 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,205 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:23:35,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:23:35,211 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:23:35,212 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce 2024-12-04T08:23:35,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:23:35,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:23:35,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:23:35,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:23:35,241 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:23:35,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:23:35,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:23:35,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:23:35,250 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:23:35,250 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:23:35,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:23:35,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:23:35,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740 2024-12-04T08:23:35,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740 2024-12-04T08:23:35,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:23:35,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:23:35,260 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44017, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:23:35,261 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:23:35,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:23:35,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42893 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,270 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42893 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,272 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:23:35,274 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=788579, jitterRate=0.0027305036783218384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:23:35,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300615235Initializing all the Stores at 1733300615237 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300615237Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300615238 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300615238Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300615238Cleaning up temporary data from old regions at 1733300615260 (+22 ms)Region opened successfully at 1733300615276 (+16 ms) 2024-12-04T08:23:35,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:23:35,277 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:23:35,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:23:35,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:23:35,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:23:35,279 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:23:35,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300615277Disabling compacts and flushes for region at 1733300615277Disabling writes for close at 1733300615277Writing region close event to WAL at 1733300615278 (+1 ms)Closed at 1733300615279 (+1 ms) 2024-12-04T08:23:35,282 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:23:35,282 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:23:35,287 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce 2024-12-04T08:23:35,288 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39713 2024-12-04T08:23:35,288 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:23:35,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:23:35,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:23:35,294 DEBUG [RS:0;f5a5a857f5c5:40167 {}] zookeeper.ZKUtil(111): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,294 WARN [RS:0;f5a5a857f5c5:40167 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:23:35,294 INFO [RS:0;f5a5a857f5c5:40167 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:23:35,294 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,296 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,40167,1733300614228] 2024-12-04T08:23:35,300 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:23:35,303 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:23:35,322 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:23:35,336 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:23:35,341 INFO [RS:0;f5a5a857f5c5:40167 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:23:35,341 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,342 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:23:35,348 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:23:35,349 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,350 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,350 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,350 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,350 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,350 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,350 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:23:35,351 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:23:35,352 DEBUG [RS:0;f5a5a857f5c5:40167 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:23:35,353 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,353 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,353 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,353 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,353 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,353 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,40167,1733300614228-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:23:35,372 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:23:35,374 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,40167,1733300614228-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,374 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,375 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.Replication(171): f5a5a857f5c5,40167,1733300614228 started 2024-12-04T08:23:35,392 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:35,392 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,40167,1733300614228, RpcServer on f5a5a857f5c5/172.17.0.2:40167, sessionid=0x1017c9258970001 2024-12-04T08:23:35,393 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:23:35,393 DEBUG [RS:0;f5a5a857f5c5:40167 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,394 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,40167,1733300614228' 2024-12-04T08:23:35,394 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:23:35,395 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:23:35,395 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:23:35,395 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:23:35,396 DEBUG [RS:0;f5a5a857f5c5:40167 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,396 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,40167,1733300614228' 2024-12-04T08:23:35,396 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:23:35,396 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:23:35,397 DEBUG [RS:0;f5a5a857f5c5:40167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:23:35,397 INFO [RS:0;f5a5a857f5c5:40167 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:23:35,397 INFO [RS:0;f5a5a857f5c5:40167 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:23:35,455 WARN [f5a5a857f5c5:42893 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:23:35,506 INFO [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C40167%2C1733300614228, suffix=, logDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228, archiveDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs, maxLogs=32 2024-12-04T08:23:35,508 INFO [RS:0;f5a5a857f5c5:40167 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300615508 2024-12-04T08:23:35,517 INFO [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300615508 2024-12-04T08:23:35,522 DEBUG [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:23:35,707 DEBUG [f5a5a857f5c5:42893 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:23:35,720 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:35,727 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,40167,1733300614228, state=OPENING 2024-12-04T08:23:35,733 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:23:35,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:23:35,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:23:35,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:23:35,737 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:23:35,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,40167,1733300614228}] 2024-12-04T08:23:35,914 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:23:35,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58671, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:23:35,928 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:23:35,929 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:23:35,933 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C40167%2C1733300614228.meta, suffix=.meta, logDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228, archiveDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs, maxLogs=32 2024-12-04T08:23:35,935 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.meta.1733300615934.meta 2024-12-04T08:23:35,943 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.meta.1733300615934.meta 2024-12-04T08:23:35,945 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45747:45747),(127.0.0.1/127.0.0.1:36577:36577)] 2024-12-04T08:23:35,949 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:23:35,951 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:23:35,954 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:23:35,959 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:23:35,964 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:23:35,965 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:23:35,965 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:23:35,965 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:23:35,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:23:35,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:23:35,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:23:35,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:23:35,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:23:35,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:23:35,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:23:35,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:23:35,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:35,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:23:35,978 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:23:35,979 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740 2024-12-04T08:23:35,982 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740 2024-12-04T08:23:35,984 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:23:35,985 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:23:35,985 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:23:35,988 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:23:35,990 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=726505, jitterRate=-0.07620160281658173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:23:35,990 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:23:35,992 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300615965Writing region info on filesystem at 1733300615966 (+1 ms)Initializing all the Stores at 1733300615967 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300615968 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300615968Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300615968Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300615968Cleaning up temporary data from old regions at 1733300615985 (+17 ms)Running coprocessor post-open hooks at 1733300615990 (+5 ms)Region opened successfully at 1733300615991 (+1 ms) 2024-12-04T08:23:35,999 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300615905 2024-12-04T08:23:36,011 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:23:36,012 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:23:36,014 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:36,017 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,40167,1733300614228, state=OPEN 2024-12-04T08:23:36,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:23:36,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:23:36,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:23:36,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:23:36,026 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:36,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:23:36,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,40167,1733300614228 in 288 msec 2024-12-04T08:23:36,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:23:36,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 744 msec 2024-12-04T08:23:36,041 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:23:36,041 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:23:36,062 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:23:36,064 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,40167,1733300614228, seqNum=-1] 2024-12-04T08:23:36,092 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:23:36,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45331, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:23:36,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0180 sec 2024-12-04T08:23:36,117 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300616117, completionTime=-1 2024-12-04T08:23:36,120 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:23:36,120 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:23:36,153 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:23:36,153 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300676153 2024-12-04T08:23:36,153 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733300736153 2024-12-04T08:23:36,153 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-12-04T08:23:36,156 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42893,1733300613527-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,157 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42893,1733300613527-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,157 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42893,1733300613527-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,158 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:42893, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,159 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,159 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,165 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:23:36,190 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.816sec 2024-12-04T08:23:36,191 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:23:36,192 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:23:36,193 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:23:36,194 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:23:36,194 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:23:36,195 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42893,1733300613527-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:23:36,195 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42893,1733300613527-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:23:36,205 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:23:36,206 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:23:36,206 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42893,1733300613527-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:23:36,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4731d90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:23:36,237 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T08:23:36,237 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T08:23:36,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,42893,-1 for getting cluster id 2024-12-04T08:23:36,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:23:36,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56cf4954-f064-4471-bb83-ad6d0ad05eb9' 2024-12-04T08:23:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:23:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56cf4954-f064-4471-bb83-ad6d0ad05eb9" 2024-12-04T08:23:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305a62c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:23:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,42893,-1] 2024-12-04T08:23:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:23:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:23:36,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:23:36,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb3700d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:23:36,266 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:23:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,40167,1733300614228, seqNum=-1] 2024-12-04T08:23:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:23:36,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:23:36,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:36,298 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:23:36,307 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:23:36,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T08:23:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f5a5a857f5c5,42893,1733300613527 2024-12-04T08:23:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1329d828 2024-12-04T08:23:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T08:23:36,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T08:23:36,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T08:23:36,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T08:23:36,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:23:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-12-04T08:23:36,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T08:23:36,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-12-04T08:23:36,345 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:36,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T08:23:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:23:36,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741835_1011 (size=389) 2024-12-04T08:23:36,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741835_1011 (size=389) 2024-12-04T08:23:36,381 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7c653bfa2ae54ad0b3c488519eaa2b9, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce 2024-12-04T08:23:36,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741836_1012 (size=72) 2024-12-04T08:23:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741836_1012 (size=72) 2024-12-04T08:23:36,394 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:23:36,394 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing b7c653bfa2ae54ad0b3c488519eaa2b9, disabling compactions & flushes 2024-12-04T08:23:36,394 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,395 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,395 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. after waiting 0 ms 2024-12-04T08:23:36,395 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,395 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,395 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7c653bfa2ae54ad0b3c488519eaa2b9: Waiting for close lock at 1733300616394Disabling compacts and flushes for region at 1733300616394Disabling writes for close at 1733300616395 (+1 ms)Writing region close event to WAL at 1733300616395Closed at 1733300616395 2024-12-04T08:23:36,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T08:23:36,402 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733300616397"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300616397"}]},"ts":"1733300616397"} 2024-12-04T08:23:36,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T08:23:36,410 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T08:23:36,413 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300616410"}]},"ts":"1733300616410"} 2024-12-04T08:23:36,418 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-12-04T08:23:36,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7c653bfa2ae54ad0b3c488519eaa2b9, ASSIGN}] 2024-12-04T08:23:36,422 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7c653bfa2ae54ad0b3c488519eaa2b9, ASSIGN 2024-12-04T08:23:36,424 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7c653bfa2ae54ad0b3c488519eaa2b9, ASSIGN; state=OFFLINE, location=f5a5a857f5c5,40167,1733300614228; forceNewPlan=false, retain=false 2024-12-04T08:23:36,595 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b7c653bfa2ae54ad0b3c488519eaa2b9, regionState=OPENING, regionLocation=f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:36,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7c653bfa2ae54ad0b3c488519eaa2b9, ASSIGN because future has completed 2024-12-04T08:23:36,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7c653bfa2ae54ad0b3c488519eaa2b9, server=f5a5a857f5c5,40167,1733300614228}] 2024-12-04T08:23:36,761 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,762 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b7c653bfa2ae54ad0b3c488519eaa2b9, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:23:36,762 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,762 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:23:36,762 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,762 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,765 INFO [StoreOpener-b7c653bfa2ae54ad0b3c488519eaa2b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,767 INFO [StoreOpener-b7c653bfa2ae54ad0b3c488519eaa2b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7c653bfa2ae54ad0b3c488519eaa2b9 columnFamilyName info 2024-12-04T08:23:36,767 DEBUG [StoreOpener-b7c653bfa2ae54ad0b3c488519eaa2b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:23:36,768 INFO [StoreOpener-b7c653bfa2ae54ad0b3c488519eaa2b9-1 {}] regionserver.HStore(327): Store=b7c653bfa2ae54ad0b3c488519eaa2b9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:23:36,769 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,770 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,770 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,771 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,771 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,774 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,777 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:23:36,778 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b7c653bfa2ae54ad0b3c488519eaa2b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835380, jitterRate=0.06224152445793152}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:23:36,778 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:36,779 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b7c653bfa2ae54ad0b3c488519eaa2b9: Running coprocessor pre-open hook at 1733300616762Writing region info on filesystem at 1733300616762Initializing all the Stores at 1733300616764 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300616764Cleaning up temporary data from old regions at 1733300616771 (+7 ms)Running coprocessor post-open hooks at 1733300616778 (+7 ms)Region opened successfully at 1733300616779 (+1 ms) 2024-12-04T08:23:36,781 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9., pid=6, masterSystemTime=1733300616755 2024-12-04T08:23:36,785 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,785 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:36,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b7c653bfa2ae54ad0b3c488519eaa2b9, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,40167,1733300614228 2024-12-04T08:23:36,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7c653bfa2ae54ad0b3c488519eaa2b9, server=f5a5a857f5c5,40167,1733300614228 because future has completed 2024-12-04T08:23:36,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T08:23:36,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b7c653bfa2ae54ad0b3c488519eaa2b9, server=f5a5a857f5c5,40167,1733300614228 in 191 msec 2024-12-04T08:23:36,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T08:23:36,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=b7c653bfa2ae54ad0b3c488519eaa2b9, ASSIGN in 377 msec 2024-12-04T08:23:36,802 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T08:23:36,803 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300616802"}]},"ts":"1733300616802"} 2024-12-04T08:23:36,806 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-12-04T08:23:36,807 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T08:23:36,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 474 msec 2024-12-04T08:23:41,470 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T08:23:41,523 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T08:23:41,525 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-12-04T08:23:43,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T08:23:43,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T08:23:43,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-04T08:23:43,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T08:23:43,936 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:23:43,936 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T08:23:43,937 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T08:23:43,937 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T08:23:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42893 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:23:46,422 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-12-04T08:23:46,425 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-12-04T08:23:46,434 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-12-04T08:23:46,435 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:23:46,436 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300626435 2024-12-04T08:23:46,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:23:46,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:23:46,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:23:46,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:23:46,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:23:46,463 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300615508 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300626435 2024-12-04T08:23:46,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741833_1009 (size=451) 2024-12-04T08:23:46,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741833_1009 (size=451) 2024-12-04T08:23:46,473 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300615508 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300615508 2024-12-04T08:23:46,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:23:46,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9., hostname=f5a5a857f5c5,40167,1733300614228, seqNum=2] 2024-12-04T08:23:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40167 {}] regionserver.HRegion(8855): Flush requested on b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:23:58,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7c653bfa2ae54ad0b3c488519eaa2b9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:23:58,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/558f437e040349108760ab10fd5aa03b is 1080, key is row0001/info:/1733300626486/Put/seqid=0 2024-12-04T08:23:58,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741838_1014 (size=12509) 2024-12-04T08:23:58,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741838_1014 (size=12509) 2024-12-04T08:23:58,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/558f437e040349108760ab10fd5aa03b 2024-12-04T08:23:58,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/558f437e040349108760ab10fd5aa03b as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b 2024-12-04T08:23:58,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b, entries=7, sequenceid=11, filesize=12.2 K 2024-12-04T08:23:58,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7c653bfa2ae54ad0b3c488519eaa2b9 in 151ms, sequenceid=11, compaction requested=false 2024-12-04T08:23:58,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7c653bfa2ae54ad0b3c488519eaa2b9: 2024-12-04T08:24:02,533 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:24:06,529 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300646529 2024-12-04T08:24:06,738 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:06,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:06,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:06,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:06,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:06,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:06,739 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300626435 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300646529 2024-12-04T08:24:06,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:24:06,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300626435 is not closed yet, will try archiving it next time 2024-12-04T08:24:06,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741837_1013 (size=12399) 2024-12-04T08:24:06,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741837_1013 (size=12399) 2024-12-04T08:24:06,944 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:09,148 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:11,352 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:13,557 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40167 {}] regionserver.HRegion(8855): Flush requested on b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:24:13,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7c653bfa2ae54ad0b3c488519eaa2b9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:24:13,759 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:13,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/c9f128c6565e486dbe22bcfa919e2899 is 1080, key is row0008/info:/1733300640518/Put/seqid=0 2024-12-04T08:24:13,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741840_1016 (size=12509) 2024-12-04T08:24:13,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741840_1016 (size=12509) 2024-12-04T08:24:13,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/c9f128c6565e486dbe22bcfa919e2899 2024-12-04T08:24:13,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/c9f128c6565e486dbe22bcfa919e2899 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/c9f128c6565e486dbe22bcfa919e2899 2024-12-04T08:24:13,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/c9f128c6565e486dbe22bcfa919e2899, entries=7, sequenceid=21, filesize=12.2 K 2024-12-04T08:24:13,995 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:13,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7c653bfa2ae54ad0b3c488519eaa2b9 in 437ms, sequenceid=21, compaction requested=false 2024-12-04T08:24:13,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7c653bfa2ae54ad0b3c488519eaa2b9: 2024-12-04T08:24:13,995 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-12-04T08:24:13,995 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:24:13,996 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b because midkey is the same as first or last row 2024-12-04T08:24:15,761 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:16,581 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T08:24:16,581 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T08:24:17,965 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:17,967 WARN [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:17,968 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C40167%2C1733300614228:(num 1733300646529) roll requested 2024-12-04T08:24:17,969 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300657968 2024-12-04T08:24:18,177 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:18,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:18,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:18,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:18,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:18,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:18,178 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300646529 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300657968 2024-12-04T08:24:18,180 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45747:45747),(127.0.0.1/127.0.0.1:36577:36577)] 2024-12-04T08:24:18,181 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300646529 is not closed yet, will try archiving it next time 2024-12-04T08:24:18,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741839_1015 (size=7739) 2024-12-04T08:24:18,181 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300626435 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300626435 2024-12-04T08:24:18,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741839_1015 (size=7739) 2024-12-04T08:24:20,169 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:21,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b7c653bfa2ae54ad0b3c488519eaa2b9, had cached 0 bytes from a total of 25018 2024-12-04T08:24:22,374 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:24,580 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:26,784 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:28,786 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T08:24:28,787 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300668786 2024-12-04T08:24:32,533 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:24:33,795 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:33,797 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:33,797 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C40167%2C1733300614228:(num 1733300668786) roll requested 2024-12-04T08:24:33,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:33,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:33,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:33,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:33,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:33,798 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300657968 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300668786 2024-12-04T08:24:33,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:24:33,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300657968 is not closed yet, will try archiving it next time 2024-12-04T08:24:33,800 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300673799 2024-12-04T08:24:33,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741841_1017 (size=4753) 2024-12-04T08:24:33,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741841_1017 (size=4753) 2024-12-04T08:24:38,803 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:38,803 WARN [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40167 {}] regionserver.HRegion(8855): Flush requested on b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:24:38,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7c653bfa2ae54ad0b3c488519eaa2b9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:24:38,810 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:38,811 WARN [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:40,804 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T08:24:43,805 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:43,805 WARN [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK], DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK]] 2024-12-04T08:24:43,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:43,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:43,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:43,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:43,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:43,806 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300668786 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300673799 2024-12-04T08:24:43,807 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45747:45747),(127.0.0.1/127.0.0.1:36577:36577)] 2024-12-04T08:24:43,808 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300668786 is not closed yet, will try archiving it next time 2024-12-04T08:24:43,808 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C40167%2C1733300614228:(num 1733300673799) roll requested 2024-12-04T08:24:43,808 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300683808 2024-12-04T08:24:43,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741842_1018 (size=1569) 2024-12-04T08:24:43,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741842_1018 (size=1569) 2024-12-04T08:24:43,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/2db63dd5b3d54b5d9d65fb5cda8366d2 is 1080, key is row0015/info:/1733300655559/Put/seqid=0 2024-12-04T08:24:43,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741844_1020 (size=12509) 2024-12-04T08:24:43,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741844_1020 (size=12509) 2024-12-04T08:24:43,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/2db63dd5b3d54b5d9d65fb5cda8366d2 2024-12-04T08:24:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/2db63dd5b3d54b5d9d65fb5cda8366d2 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/2db63dd5b3d54b5d9d65fb5cda8366d2 2024-12-04T08:24:43,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/2db63dd5b3d54b5d9d65fb5cda8366d2, entries=7, sequenceid=31, filesize=12.2 K 2024-12-04T08:24:48,819 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:48,819 WARN [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:48,840 INFO [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:48,840 WARN [FSHLog-0-hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce-prefix:f5a5a857f5c5,40167,1733300614228 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44543,DS-137e4273-5e71-45b3-8a2f-e69da349b8b9,DISK], DatanodeInfoWithStorage[127.0.0.1:38971,DS-12ed9b94-4d1d-4bdb-9725-0ed0f1499fe1,DISK]] 2024-12-04T08:24:48,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7c653bfa2ae54ad0b3c488519eaa2b9 in 10037ms, sequenceid=31, compaction requested=true 2024-12-04T08:24:48,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7c653bfa2ae54ad0b3c488519eaa2b9: 2024-12-04T08:24:48,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,841 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-12-04T08:24:48,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,841 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:24:48,841 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b because midkey is the same as first or last row 2024-12-04T08:24:48,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,841 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300673799 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300683808 2024-12-04T08:24:48,842 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:24:48,842 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300673799 is not closed yet, will try archiving it next time 2024-12-04T08:24:48,843 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300646529 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300646529 2024-12-04T08:24:48,843 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C40167%2C1733300614228:(num 1733300683808) roll requested 2024-12-04T08:24:48,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7c653bfa2ae54ad0b3c488519eaa2b9:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:24:48,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300688843 2024-12-04T08:24:48,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741843_1019 (size=438) 2024-12-04T08:24:48,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741843_1019 (size=438) 2024-12-04T08:24:48,845 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300657968 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300657968 2024-12-04T08:24:48,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:24:48,847 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:24:48,847 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300668786 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300668786 2024-12-04T08:24:48,848 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300673799 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300673799 2024-12-04T08:24:48,850 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:24:48,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,851 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.HStore(1541): b7c653bfa2ae54ad0b3c488519eaa2b9/info is initiating minor compaction (all files) 2024-12-04T08:24:48,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,852 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300683808 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300688843 2024-12-04T08:24:48,852 INFO [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7c653bfa2ae54ad0b3c488519eaa2b9/info in TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:24:48,852 INFO [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/c9f128c6565e486dbe22bcfa919e2899, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/2db63dd5b3d54b5d9d65fb5cda8366d2] into tmpdir=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp, totalSize=36.6 K 2024-12-04T08:24:48,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577),(127.0.0.1/127.0.0.1:45747:45747)] 2024-12-04T08:24:48,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300683808 is not closed yet, will try archiving it next time 2024-12-04T08:24:48,853 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C40167%2C1733300614228.1733300688853 2024-12-04T08:24:48,854 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 558f437e040349108760ab10fd5aa03b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733300626486 2024-12-04T08:24:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741845_1021 (size=93) 2024-12-04T08:24:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741845_1021 (size=93) 2024-12-04T08:24:48,855 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f128c6565e486dbe22bcfa919e2899, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733300640518 2024-12-04T08:24:48,855 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300683808 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs/f5a5a857f5c5%2C40167%2C1733300614228.1733300683808 2024-12-04T08:24:48,855 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db63dd5b3d54b5d9d65fb5cda8366d2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733300655559 2024-12-04T08:24:48,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:24:48,863 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300688843 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300688853 2024-12-04T08:24:48,865 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45747:45747),(127.0.0.1/127.0.0.1:36577:36577)] 2024-12-04T08:24:48,865 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/WALs/f5a5a857f5c5,40167,1733300614228/f5a5a857f5c5%2C40167%2C1733300614228.1733300688843 is not closed yet, will try archiving it next time 2024-12-04T08:24:48,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741846_1022 (size=1258) 2024-12-04T08:24:48,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741846_1022 (size=1258) 2024-12-04T08:24:48,886 INFO [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7c653bfa2ae54ad0b3c488519eaa2b9#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:24:48,887 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/017675d4b4b94ef4975e77f2150f1bc3 is 1080, key is row0001/info:/1733300626486/Put/seqid=0 2024-12-04T08:24:48,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741848_1024 (size=27710) 2024-12-04T08:24:48,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741848_1024 (size=27710) 2024-12-04T08:24:48,906 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/017675d4b4b94ef4975e77f2150f1bc3 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/017675d4b4b94ef4975e77f2150f1bc3 2024-12-04T08:24:48,922 INFO [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7c653bfa2ae54ad0b3c488519eaa2b9/info of b7c653bfa2ae54ad0b3c488519eaa2b9 into 017675d4b4b94ef4975e77f2150f1bc3(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:24:48,922 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7c653bfa2ae54ad0b3c488519eaa2b9: 2024-12-04T08:24:48,924 INFO [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9., storeName=b7c653bfa2ae54ad0b3c488519eaa2b9/info, priority=13, startTime=1733300688843; duration=0sec 2024-12-04T08:24:48,924 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-04T08:24:48,924 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:24:48,924 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/017675d4b4b94ef4975e77f2150f1bc3 because midkey is the same as first or last row 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/017675d4b4b94ef4975e77f2150f1bc3 because midkey is the same as first or last row 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/017675d4b4b94ef4975e77f2150f1bc3 because midkey is the same as first or last row 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:24:48,925 DEBUG [RS:0;f5a5a857f5c5:40167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7c653bfa2ae54ad0b3c488519eaa2b9:info 2024-12-04T08:25:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40167 {}] regionserver.HRegion(8855): Flush requested on b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:25:00,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7c653bfa2ae54ad0b3c488519eaa2b9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:25:00,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/382ed429131d449694c45ef57ba7cb5e is 1080, key is row0022/info:/1733300688855/Put/seqid=0 2024-12-04T08:25:00,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741849_1025 (size=12509) 2024-12-04T08:25:00,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741849_1025 (size=12509) 2024-12-04T08:25:00,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/382ed429131d449694c45ef57ba7cb5e 2024-12-04T08:25:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/382ed429131d449694c45ef57ba7cb5e as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/382ed429131d449694c45ef57ba7cb5e 2024-12-04T08:25:00,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/382ed429131d449694c45ef57ba7cb5e, entries=7, sequenceid=42, filesize=12.2 K 2024-12-04T08:25:00,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for b7c653bfa2ae54ad0b3c488519eaa2b9 in 34ms, sequenceid=42, compaction requested=false 2024-12-04T08:25:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7c653bfa2ae54ad0b3c488519eaa2b9: 2024-12-04T08:25:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-12-04T08:25:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:00,912 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/017675d4b4b94ef4975e77f2150f1bc3 because midkey is the same as first or last row 2024-12-04T08:25:02,533 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:25:06,763 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b7c653bfa2ae54ad0b3c488519eaa2b9, had cached 0 bytes from a total of 40219 2024-12-04T08:25:08,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:25:08,891 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:25:08,891 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:08,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:08,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:08,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:25:08,898 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:25:08,898 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1075752841, stopped=false 2024-12-04T08:25:08,898 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,42893,1733300613527 2024-12-04T08:25:08,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:08,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:08,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:08,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:08,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:25:08,901 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:25:08,902 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:08,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:08,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:08,902 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,40167,1733300614228' ***** 2024-12-04T08:25:08,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:08,902 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:25:08,903 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:25:08,903 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:25:08,903 INFO [RS:0;f5a5a857f5c5:40167 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:25:08,903 INFO [RS:0;f5a5a857f5c5:40167 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:25:08,904 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(3091): Received CLOSE for b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:25:08,904 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,40167,1733300614228 2024-12-04T08:25:08,904 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:08,904 INFO [RS:0;f5a5a857f5c5:40167 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:40167. 2024-12-04T08:25:08,905 DEBUG [RS:0;f5a5a857f5c5:40167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:08,905 DEBUG [RS:0;f5a5a857f5c5:40167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:08,905 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7c653bfa2ae54ad0b3c488519eaa2b9, disabling compactions & flushes 2024-12-04T08:25:08,905 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:25:08,905 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:25:08,905 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:25:08,905 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:25:08,905 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. after waiting 0 ms 2024-12-04T08:25:08,905 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:25:08,905 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:25:08,905 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:25:08,906 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b7c653bfa2ae54ad0b3c488519eaa2b9 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-12-04T08:25:08,906 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T08:25:08,906 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, b7c653bfa2ae54ad0b3c488519eaa2b9=TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.} 2024-12-04T08:25:08,906 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:25:08,906 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:25:08,906 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:25:08,906 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b7c653bfa2ae54ad0b3c488519eaa2b9 2024-12-04T08:25:08,906 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:25:08,906 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:25:08,907 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-12-04T08:25:08,913 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/fbc2a36d6d904736aab8b56ba8ff4906 is 1080, key is row0029/info:/1733300702880/Put/seqid=0 2024-12-04T08:25:08,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741850_1026 (size=8193) 2024-12-04T08:25:08,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741850_1026 (size=8193) 2024-12-04T08:25:08,932 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/fbc2a36d6d904736aab8b56ba8ff4906 2024-12-04T08:25:08,937 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/info/dde3b4a79ef14cf2b64489800d282a65 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9./info:regioninfo/1733300616786/Put/seqid=0 2024-12-04T08:25:08,947 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/.tmp/info/fbc2a36d6d904736aab8b56ba8ff4906 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/fbc2a36d6d904736aab8b56ba8ff4906 2024-12-04T08:25:08,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741851_1027 (size=7016) 2024-12-04T08:25:08,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741851_1027 (size=7016) 2024-12-04T08:25:08,953 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/info/dde3b4a79ef14cf2b64489800d282a65 2024-12-04T08:25:08,961 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/fbc2a36d6d904736aab8b56ba8ff4906, entries=3, sequenceid=48, filesize=8.0 K 2024-12-04T08:25:08,964 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for b7c653bfa2ae54ad0b3c488519eaa2b9 in 59ms, sequenceid=48, compaction requested=true 2024-12-04T08:25:08,967 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/c9f128c6565e486dbe22bcfa919e2899, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/2db63dd5b3d54b5d9d65fb5cda8366d2] to archive 2024-12-04T08:25:08,971 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T08:25:08,982 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/archive/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/558f437e040349108760ab10fd5aa03b 2024-12-04T08:25:08,985 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/c9f128c6565e486dbe22bcfa919e2899 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/archive/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/c9f128c6565e486dbe22bcfa919e2899 2024-12-04T08:25:08,990 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/2db63dd5b3d54b5d9d65fb5cda8366d2 to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/archive/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/info/2db63dd5b3d54b5d9d65fb5cda8366d2 2024-12-04T08:25:08,992 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/ns/5c47c73f6baf4fb9b2d4e1e5d7377750 is 43, key is default/ns:d/1733300616099/Put/seqid=0 2024-12-04T08:25:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741852_1028 (size=5153) 2024-12-04T08:25:09,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741852_1028 (size=5153) 2024-12-04T08:25:09,012 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/ns/5c47c73f6baf4fb9b2d4e1e5d7377750 2024-12-04T08:25:09,007 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f5a5a857f5c5:42893 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T08:25:09,013 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [558f437e040349108760ab10fd5aa03b=12509, c9f128c6565e486dbe22bcfa919e2899=12509, 2db63dd5b3d54b5d9d65fb5cda8366d2=12509] 2024-12-04T08:25:09,019 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/default/TestLogRolling-testSlowSyncLogRolling/b7c653bfa2ae54ad0b3c488519eaa2b9/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-12-04T08:25:09,023 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:25:09,023 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7c653bfa2ae54ad0b3c488519eaa2b9: Waiting for close lock at 1733300708904Running coprocessor pre-close hooks at 1733300708905 (+1 ms)Disabling compacts and flushes for region at 1733300708905Disabling writes for close at 1733300708905Obtaining lock to block concurrent updates at 1733300708906 (+1 ms)Preparing flush snapshotting stores in b7c653bfa2ae54ad0b3c488519eaa2b9 at 1733300708906Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733300708906Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. at 1733300708907 (+1 ms)Flushing b7c653bfa2ae54ad0b3c488519eaa2b9/info: creating writer at 1733300708908 (+1 ms)Flushing b7c653bfa2ae54ad0b3c488519eaa2b9/info: appending metadata at 1733300708913 (+5 ms)Flushing b7c653bfa2ae54ad0b3c488519eaa2b9/info: closing flushed file at 1733300708913Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@660a2fc3: reopening flushed file at 1733300708945 (+32 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for b7c653bfa2ae54ad0b3c488519eaa2b9 in 59ms, sequenceid=48, compaction requested=true at 1733300708964 (+19 ms)Writing region close event to WAL at 1733300709014 (+50 ms)Running coprocessor post-close hooks at 1733300709020 (+6 ms)Closed at 1733300709023 (+3 ms) 2024-12-04T08:25:09,024 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733300616326.b7c653bfa2ae54ad0b3c488519eaa2b9. 2024-12-04T08:25:09,053 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/table/8e9d56895b9c4d92bc8ddde578fadeef is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733300616802/Put/seqid=0 2024-12-04T08:25:09,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741853_1029 (size=5396) 2024-12-04T08:25:09,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741853_1029 (size=5396) 2024-12-04T08:25:09,062 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/table/8e9d56895b9c4d92bc8ddde578fadeef 2024-12-04T08:25:09,073 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/info/dde3b4a79ef14cf2b64489800d282a65 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/info/dde3b4a79ef14cf2b64489800d282a65 2024-12-04T08:25:09,085 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/info/dde3b4a79ef14cf2b64489800d282a65, entries=10, sequenceid=11, filesize=6.9 K 2024-12-04T08:25:09,086 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/ns/5c47c73f6baf4fb9b2d4e1e5d7377750 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/ns/5c47c73f6baf4fb9b2d4e1e5d7377750 2024-12-04T08:25:09,096 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/ns/5c47c73f6baf4fb9b2d4e1e5d7377750, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T08:25:09,098 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/.tmp/table/8e9d56895b9c4d92bc8ddde578fadeef as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/table/8e9d56895b9c4d92bc8ddde578fadeef 2024-12-04T08:25:09,107 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/table/8e9d56895b9c4d92bc8ddde578fadeef, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T08:25:09,107 DEBUG [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T08:25:09,109 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 202ms, sequenceid=11, compaction requested=false 2024-12-04T08:25:09,121 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T08:25:09,122 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:25:09,123 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:09,123 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300708906Running coprocessor pre-close hooks at 1733300708906Disabling compacts and flushes for region at 1733300708906Disabling writes for close at 1733300708906Obtaining lock to block concurrent updates at 1733300708907 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733300708907Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733300708907Flushing stores of hbase:meta,,1.1588230740 at 1733300708909 (+2 ms)Flushing 1588230740/info: creating writer at 1733300708909Flushing 1588230740/info: appending metadata at 1733300708936 (+27 ms)Flushing 1588230740/info: closing flushed file at 1733300708937 (+1 ms)Flushing 1588230740/ns: creating writer at 1733300708965 (+28 ms)Flushing 1588230740/ns: appending metadata at 1733300708991 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1733300708991Flushing 1588230740/table: creating writer at 1733300709021 (+30 ms)Flushing 1588230740/table: appending metadata at 1733300709053 (+32 ms)Flushing 1588230740/table: closing flushed file at 1733300709053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59cf2ee1: reopening flushed file at 1733300709071 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@779d1d20: reopening flushed file at 1733300709085 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f36e05a: reopening flushed file at 1733300709096 (+11 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 202ms, sequenceid=11, compaction requested=false at 1733300709109 (+13 ms)Writing region close event to WAL at 1733300709110 (+1 ms)Running coprocessor post-close hooks at 1733300709122 (+12 ms)Closed at 1733300709122 2024-12-04T08:25:09,123 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:09,307 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,40167,1733300614228; all regions closed. 2024-12-04T08:25:09,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741834_1010 (size=3066) 2024-12-04T08:25:09,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741834_1010 (size=3066) 2024-12-04T08:25:09,317 DEBUG [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs 2024-12-04T08:25:09,317 INFO [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C40167%2C1733300614228.meta:.meta(num 1733300615934) 2024-12-04T08:25:09,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741847_1023 (size=12695) 2024-12-04T08:25:09,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741847_1023 (size=12695) 2024-12-04T08:25:09,327 DEBUG [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/oldWALs 2024-12-04T08:25:09,327 INFO [RS:0;f5a5a857f5c5:40167 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C40167%2C1733300614228:(num 1733300688853) 2024-12-04T08:25:09,327 DEBUG [RS:0;f5a5a857f5c5:40167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:09,327 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:09,327 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:09,327 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:09,328 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:09,328 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:09,328 INFO [RS:0;f5a5a857f5c5:40167 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40167 2024-12-04T08:25:09,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,40167,1733300614228 2024-12-04T08:25:09,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:09,333 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:09,335 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,40167,1733300614228] 2024-12-04T08:25:09,337 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,40167,1733300614228 already deleted, retry=false 2024-12-04T08:25:09,337 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,40167,1733300614228 expired; onlineServers=0 2024-12-04T08:25:09,338 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,42893,1733300613527' ***** 2024-12-04T08:25:09,338 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:25:09,338 INFO [M:0;f5a5a857f5c5:42893 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:09,338 INFO [M:0;f5a5a857f5c5:42893 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:09,338 DEBUG [M:0;f5a5a857f5c5:42893 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:25:09,338 DEBUG [M:0;f5a5a857f5c5:42893 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:25:09,338 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:25:09,338 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300615202 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300615202,5,FailOnTimeoutGroup] 2024-12-04T08:25:09,338 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300615199 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300615199,5,FailOnTimeoutGroup] 2024-12-04T08:25:09,338 INFO [M:0;f5a5a857f5c5:42893 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:09,338 INFO [M:0;f5a5a857f5c5:42893 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:09,339 DEBUG [M:0;f5a5a857f5c5:42893 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:25:09,339 INFO [M:0;f5a5a857f5c5:42893 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:25:09,339 INFO [M:0;f5a5a857f5c5:42893 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:25:09,339 INFO [M:0;f5a5a857f5c5:42893 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:25:09,339 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:25:09,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:25:09,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:09,340 DEBUG [M:0;f5a5a857f5c5:42893 {}] zookeeper.ZKUtil(347): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:25:09,340 WARN [M:0;f5a5a857f5c5:42893 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:25:09,341 INFO [M:0;f5a5a857f5c5:42893 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/.lastflushedseqids 2024-12-04T08:25:09,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741854_1030 (size=130) 2024-12-04T08:25:09,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741854_1030 (size=130) 2024-12-04T08:25:09,354 INFO [M:0;f5a5a857f5c5:42893 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:25:09,354 INFO [M:0;f5a5a857f5c5:42893 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:25:09,354 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:25:09,354 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:09,354 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:09,354 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:25:09,354 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:09,355 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB 2024-12-04T08:25:09,358 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:09,373 DEBUG [M:0;f5a5a857f5c5:42893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ec8dec77cf444a82a0b8f26d044efe6a is 82, key is hbase:meta,,1/info:regioninfo/1733300616013/Put/seqid=0 2024-12-04T08:25:09,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741855_1031 (size=5672) 2024-12-04T08:25:09,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741855_1031 (size=5672) 2024-12-04T08:25:09,380 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ec8dec77cf444a82a0b8f26d044efe6a 2024-12-04T08:25:09,409 DEBUG [M:0;f5a5a857f5c5:42893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5bdffdea1473456a82d6a498edfcfb46 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733300616809/Put/seqid=0 2024-12-04T08:25:09,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741856_1032 (size=6248) 2024-12-04T08:25:09,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741856_1032 (size=6248) 2024-12-04T08:25:09,416 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5bdffdea1473456a82d6a498edfcfb46 2024-12-04T08:25:09,422 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5bdffdea1473456a82d6a498edfcfb46 2024-12-04T08:25:09,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:09,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40167-0x1017c9258970001, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:09,436 INFO [RS:0;f5a5a857f5c5:40167 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:09,436 INFO [RS:0;f5a5a857f5c5:40167 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,40167,1733300614228; zookeeper connection closed. 2024-12-04T08:25:09,436 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@365a2b28 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@365a2b28 2024-12-04T08:25:09,437 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T08:25:09,445 DEBUG [M:0;f5a5a857f5c5:42893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a70c97a4f4f4c2bb60a8b416228a4df is 69, key is f5a5a857f5c5,40167,1733300614228/rs:state/1733300615273/Put/seqid=0 2024-12-04T08:25:09,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741857_1033 (size=5156) 2024-12-04T08:25:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741857_1033 (size=5156) 2024-12-04T08:25:09,453 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a70c97a4f4f4c2bb60a8b416228a4df 2024-12-04T08:25:09,477 DEBUG [M:0;f5a5a857f5c5:42893 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/65a5f4dde8e04565bdaf1e63a10d1957 is 52, key is load_balancer_on/state:d/1733300616303/Put/seqid=0 2024-12-04T08:25:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741858_1034 (size=5056) 2024-12-04T08:25:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741858_1034 (size=5056) 2024-12-04T08:25:09,483 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/65a5f4dde8e04565bdaf1e63a10d1957 2024-12-04T08:25:09,491 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ec8dec77cf444a82a0b8f26d044efe6a as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ec8dec77cf444a82a0b8f26d044efe6a 2024-12-04T08:25:09,502 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ec8dec77cf444a82a0b8f26d044efe6a, entries=8, sequenceid=59, filesize=5.5 K 2024-12-04T08:25:09,505 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5bdffdea1473456a82d6a498edfcfb46 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5bdffdea1473456a82d6a498edfcfb46 2024-12-04T08:25:09,514 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5bdffdea1473456a82d6a498edfcfb46 2024-12-04T08:25:09,514 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5bdffdea1473456a82d6a498edfcfb46, entries=6, sequenceid=59, filesize=6.1 K 2024-12-04T08:25:09,516 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a70c97a4f4f4c2bb60a8b416228a4df as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a70c97a4f4f4c2bb60a8b416228a4df 2024-12-04T08:25:09,526 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a70c97a4f4f4c2bb60a8b416228a4df, entries=1, sequenceid=59, filesize=5.0 K 2024-12-04T08:25:09,528 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/65a5f4dde8e04565bdaf1e63a10d1957 as hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/65a5f4dde8e04565bdaf1e63a10d1957 2024-12-04T08:25:09,539 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/65a5f4dde8e04565bdaf1e63a10d1957, entries=1, sequenceid=59, filesize=4.9 K 2024-12-04T08:25:09,541 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 186ms, sequenceid=59, compaction requested=false 2024-12-04T08:25:09,543 INFO [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:09,543 DEBUG [M:0;f5a5a857f5c5:42893 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300709354Disabling compacts and flushes for region at 1733300709354Disabling writes for close at 1733300709354Obtaining lock to block concurrent updates at 1733300709355 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300709355Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1733300709355Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300709356 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300709356Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300709372 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300709372Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300709389 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300709408 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300709408Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300709422 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300709445 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300709445Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300709460 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300709476 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300709476Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29330c03: reopening flushed file at 1733300709489 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e7355b4: reopening flushed file at 1733300709502 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59567ac3: reopening flushed file at 1733300709515 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208e6e04: reopening flushed file at 1733300709526 (+11 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 186ms, sequenceid=59, compaction requested=false at 1733300709541 (+15 ms)Writing region close event to WAL at 1733300709543 (+2 ms)Closed at 1733300709543 2024-12-04T08:25:09,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:09,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44543 is added to blk_1073741830_1006 (size=27985) 2024-12-04T08:25:09,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38971 is added to blk_1073741830_1006 (size=27985) 2024-12-04T08:25:09,548 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:09,548 INFO [M:0;f5a5a857f5c5:42893 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:25:09,549 INFO [M:0;f5a5a857f5c5:42893 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42893 2024-12-04T08:25:09,549 INFO [M:0;f5a5a857f5c5:42893 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:09,651 INFO [M:0;f5a5a857f5c5:42893 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42893-0x1017c9258970000, quorum=127.0.0.1:51944, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:09,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:09,661 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:09,661 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:09,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:09,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:09,666 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:09,666 WARN [BP-864805852-172.17.0.2-1733300610521 heartbeating to localhost/127.0.0.1:39713 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:09,666 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:09,666 WARN [BP-864805852-172.17.0.2-1733300610521 heartbeating to localhost/127.0.0.1:39713 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-864805852-172.17.0.2-1733300610521 (Datanode Uuid 621430fd-59eb-4131-9539-9e1f04696325) service to localhost/127.0.0.1:39713 2024-12-04T08:25:09,668 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data3/current/BP-864805852-172.17.0.2-1733300610521 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:09,668 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data4/current/BP-864805852-172.17.0.2-1733300610521 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:09,669 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:09,680 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:09,681 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:09,681 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:09,681 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:09,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:09,684 WARN [BP-864805852-172.17.0.2-1733300610521 heartbeating to localhost/127.0.0.1:39713 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:09,684 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:09,684 WARN [BP-864805852-172.17.0.2-1733300610521 heartbeating to localhost/127.0.0.1:39713 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-864805852-172.17.0.2-1733300610521 (Datanode Uuid c3f2aa86-8653-489c-aa20-7708c4d2db01) service to localhost/127.0.0.1:39713 2024-12-04T08:25:09,684 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:09,684 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data1/current/BP-864805852-172.17.0.2-1733300610521 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:09,685 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/cluster_422f5dc4-7a41-e4c6-8101-ed413782d3c3/data/data2/current/BP-864805852-172.17.0.2-1733300610521 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:09,685 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:09,704 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:25:09,705 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:09,705 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:09,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:09,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:09,718 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:25:09,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:25:09,782 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=80 (was 12) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: master/f5a5a857f5c5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f5a5a857f5c5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: LeaseRenewer:jenkins@localhost:39713 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39713 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39713 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39713 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f5a5a857f5c5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39713 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@4e6bf3cd java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39713 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39713 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:39713 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=115 (was 245), ProcessCount=11 (was 11), AvailableMemoryMB=7329 (was 7167) - AvailableMemoryMB LEAK? - 2024-12-04T08:25:09,792 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=81, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=115, ProcessCount=11, AvailableMemoryMB=7328 2024-12-04T08:25:09,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:25:09,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.log.dir so I do NOT create it in target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a 2024-12-04T08:25:09,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84693f0f-284e-c151-4e99-473e7e1ff30d/hadoop.tmp.dir so I do NOT create it in target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a 2024-12-04T08:25:09,793 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6, deleteOnExit=true 2024-12-04T08:25:09,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:25:09,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/test.cache.data in system properties and HBase conf 2024-12-04T08:25:09,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:25:09,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:25:09,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:25:09,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:25:09,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:25:09,794 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:25:09,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:25:09,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:25:09,819 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:25:09,915 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:09,921 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:09,922 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:09,922 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:09,922 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:09,923 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:09,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@630e1a46{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:09,924 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d9de743{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:10,042 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bc50e3b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/java.io.tmpdir/jetty-localhost-38775-hadoop-hdfs-3_4_1-tests_jar-_-any-13313336567818617016/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:25:10,043 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8c19991{HTTP/1.1, (http/1.1)}{localhost:38775} 2024-12-04T08:25:10,043 INFO [Time-limited test {}] server.Server(415): Started @101456ms 2024-12-04T08:25:10,057 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:25:10,131 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:10,135 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:10,135 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:10,135 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:10,135 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:25:10,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3270c9ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:10,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4620cd8a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:10,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@78fa6004{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/java.io.tmpdir/jetty-localhost-46239-hadoop-hdfs-3_4_1-tests_jar-_-any-18233878173175096732/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:10,254 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6b5fac92{HTTP/1.1, (http/1.1)}{localhost:46239} 2024-12-04T08:25:10,254 INFO [Time-limited test {}] server.Server(415): Started @101668ms 2024-12-04T08:25:10,256 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:10,303 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:10,308 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:10,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:10,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:10,309 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:25:10,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c30f553{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:10,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ab06e68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:10,360 WARN [Thread-440 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data2/current/BP-827450206-172.17.0.2-1733300709842/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:10,360 WARN [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data1/current/BP-827450206-172.17.0.2-1733300709842/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:10,381 WARN [Thread-418 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:10,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x590d9b80239bce47 with lease ID 0x8213aefcb027233d: Processing first storage report for DS-cf35e3e0-6611-42f9-b776-daa07adccf45 from datanode DatanodeRegistration(127.0.0.1:45807, datanodeUuid=d9bd77ee-49d0-4ed3-b636-58e05a737b53, infoPort=42635, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842) 2024-12-04T08:25:10,388 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x590d9b80239bce47 with lease ID 0x8213aefcb027233d: from storage DS-cf35e3e0-6611-42f9-b776-daa07adccf45 node DatanodeRegistration(127.0.0.1:45807, datanodeUuid=d9bd77ee-49d0-4ed3-b636-58e05a737b53, infoPort=42635, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T08:25:10,388 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x590d9b80239bce47 with lease ID 0x8213aefcb027233d: Processing first storage report for DS-d188b1f0-f924-4c2c-8fa9-bcbd3ad366db from datanode DatanodeRegistration(127.0.0.1:45807, datanodeUuid=d9bd77ee-49d0-4ed3-b636-58e05a737b53, infoPort=42635, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842) 2024-12-04T08:25:10,388 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x590d9b80239bce47 with lease ID 0x8213aefcb027233d: from storage DS-d188b1f0-f924-4c2c-8fa9-bcbd3ad366db node DatanodeRegistration(127.0.0.1:45807, datanodeUuid=d9bd77ee-49d0-4ed3-b636-58e05a737b53, infoPort=42635, infoSecurePort=0, ipcPort=33691, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:10,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38d766c7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/java.io.tmpdir/jetty-localhost-36837-hadoop-hdfs-3_4_1-tests_jar-_-any-7693967167038424049/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:10,429 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@12686d25{HTTP/1.1, (http/1.1)}{localhost:36837} 2024-12-04T08:25:10,430 INFO [Time-limited test {}] server.Server(415): Started @101843ms 2024-12-04T08:25:10,432 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:10,553 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data3/current/BP-827450206-172.17.0.2-1733300709842/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:10,553 WARN [Thread-466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data4/current/BP-827450206-172.17.0.2-1733300709842/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:10,573 WARN [Thread-454 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:10,576 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84cbc89b7e18da00 with lease ID 0x8213aefcb027233e: Processing first storage report for DS-47d4e9f5-5797-4af7-8c13-fb4f2990df5a from datanode DatanodeRegistration(127.0.0.1:45939, datanodeUuid=92796feb-b164-4241-a1d3-4551808dd045, infoPort=34849, infoSecurePort=0, ipcPort=42441, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842) 2024-12-04T08:25:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84cbc89b7e18da00 with lease ID 0x8213aefcb027233e: from storage DS-47d4e9f5-5797-4af7-8c13-fb4f2990df5a node DatanodeRegistration(127.0.0.1:45939, datanodeUuid=92796feb-b164-4241-a1d3-4551808dd045, infoPort=34849, infoSecurePort=0, ipcPort=42441, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T08:25:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84cbc89b7e18da00 with lease ID 0x8213aefcb027233e: Processing first storage report for DS-afb2735f-a507-46e4-be0e-386ce76958e9 from datanode DatanodeRegistration(127.0.0.1:45939, datanodeUuid=92796feb-b164-4241-a1d3-4551808dd045, infoPort=34849, infoSecurePort=0, ipcPort=42441, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842) 2024-12-04T08:25:10,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84cbc89b7e18da00 with lease ID 0x8213aefcb027233e: from storage DS-afb2735f-a507-46e4-be0e-386ce76958e9 node DatanodeRegistration(127.0.0.1:45939, datanodeUuid=92796feb-b164-4241-a1d3-4551808dd045, infoPort=34849, infoSecurePort=0, ipcPort=42441, storageInfo=lv=-57;cid=testClusterID;nsid=588909697;c=1733300709842), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:10,668 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a 2024-12-04T08:25:10,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/zookeeper_0, clientPort=49184, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:25:10,673 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49184 2024-12-04T08:25:10,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:25:10,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:25:10,690 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d with version=8 2024-12-04T08:25:10,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:25:10,693 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:25:10,693 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:10,694 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43255 2024-12-04T08:25:10,696 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43255 connecting to ZooKeeper ensemble=127.0.0.1:49184 2024-12-04T08:25:10,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432550x0, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:10,709 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43255-0x1017c93d79f0000 connected 2024-12-04T08:25:10,735 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:10,740 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d, hbase.cluster.distributed=false 2024-12-04T08:25:10,742 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:10,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43255 2024-12-04T08:25:10,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43255 2024-12-04T08:25:10,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43255 2024-12-04T08:25:10,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43255 2024-12-04T08:25:10,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43255 2024-12-04T08:25:10,762 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:25:10,762 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:10,763 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32891 2024-12-04T08:25:10,765 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32891 connecting to ZooKeeper ensemble=127.0.0.1:49184 2024-12-04T08:25:10,765 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328910x0, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:10,773 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328910x0, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:10,773 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32891-0x1017c93d79f0001 connected 2024-12-04T08:25:10,774 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:25:10,775 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:25:10,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:25:10,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:10,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32891 2024-12-04T08:25:10,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32891 2024-12-04T08:25:10,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32891 2024-12-04T08:25:10,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32891 2024-12-04T08:25:10,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32891 2024-12-04T08:25:10,801 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:43255 2024-12-04T08:25:10,801 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:10,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:10,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:10,804 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:10,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:25:10,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,807 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:25:10,808 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,43255,1733300710692 from backup master directory 2024-12-04T08:25:10,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:10,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:10,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:10,811 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:10,811 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:10,817 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/hbase.id] with ID: 0e0ac2d2-6d59-44c5-8652-5f8ad1134681 2024-12-04T08:25:10,818 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/.tmp/hbase.id 2024-12-04T08:25:10,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:25:10,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:25:10,827 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/.tmp/hbase.id]:[hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/hbase.id] 2024-12-04T08:25:10,845 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:10,845 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:25:10,847 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-04T08:25:10,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:10,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:10,861 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:25:10,862 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:25:10,862 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:25:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:25:10,878 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store 2024-12-04T08:25:10,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:10,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:10,896 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:10,897 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:25:10,897 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:10,897 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:10,897 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:25:10,897 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:10,897 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:10,897 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300710896Disabling compacts and flushes for region at 1733300710896Disabling writes for close at 1733300710897 (+1 ms)Writing region close event to WAL at 1733300710897Closed at 1733300710897 2024-12-04T08:25:10,900 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/.initializing 2024-12-04T08:25:10,900 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/WALs/f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:10,904 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C43255%2C1733300710692, suffix=, logDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/WALs/f5a5a857f5c5,43255,1733300710692, archiveDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/oldWALs, maxLogs=10 2024-12-04T08:25:10,905 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C43255%2C1733300710692.1733300710904 2024-12-04T08:25:10,912 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/WALs/f5a5a857f5c5,43255,1733300710692/f5a5a857f5c5%2C43255%2C1733300710692.1733300710904 2024-12-04T08:25:10,913 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34849:34849),(127.0.0.1/127.0.0.1:42635:42635)] 2024-12-04T08:25:10,917 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:10,917 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:10,917 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,917 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:25:10,922 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:10,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:10,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:25:10,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:10,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:10,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,931 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:25:10,931 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:10,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:10,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:25:10,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:10,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:10,935 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,938 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,939 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,941 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,941 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,942 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:25:10,944 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:10,947 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:10,947 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785058, jitterRate=-0.0017473101615905762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:25:10,949 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300710918Initializing all the Stores at 1733300710919 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300710919Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300710920 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300710920Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300710920Cleaning up temporary data from old regions at 1733300710941 (+21 ms)Region opened successfully at 1733300710949 (+8 ms) 2024-12-04T08:25:10,951 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:25:10,958 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6286d70d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:10,959 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:25:10,960 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:25:10,960 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:25:10,960 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:25:10,961 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T08:25:10,962 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T08:25:10,962 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:25:10,968 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:25:10,969 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:25:10,970 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:25:10,971 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:25:10,972 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:25:10,973 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:25:10,974 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:25:10,977 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:25:10,980 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:25:10,981 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:25:10,982 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:25:10,985 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:25:10,986 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:25:10,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:10,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:10,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,989 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,43255,1733300710692, sessionid=0x1017c93d79f0000, setting cluster-up flag (Was=false) 2024-12-04T08:25:10,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:10,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,002 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:25:11,003 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:11,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,014 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:25:11,015 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:11,017 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:25:11,019 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:11,020 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:25:11,020 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:25:11,020 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,43255,1733300710692 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:11,023 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,029 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:11,029 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:25:11,031 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,032 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300741033 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:25:11,033 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,040 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:25:11,040 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:25:11,040 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:25:11,041 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:25:11,041 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:25:11,041 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300711041,5,FailOnTimeoutGroup] 2024-12-04T08:25:11,042 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300711042,5,FailOnTimeoutGroup] 2024-12-04T08:25:11,042 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,042 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:25:11,042 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,042 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:11,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:11,052 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:25:11,053 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d 2024-12-04T08:25:11,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:25:11,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:25:11,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:11,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:25:11,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:25:11,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:25:11,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:25:11,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:25:11,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:25:11,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,088 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(746): ClusterId : 0e0ac2d2-6d59-44c5-8652-5f8ad1134681 2024-12-04T08:25:11,088 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:25:11,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:25:11,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:25:11,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,091 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:25:11,091 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:25:11,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:25:11,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740 2024-12-04T08:25:11,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740 2024-12-04T08:25:11,094 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:25:11,094 DEBUG [RS:0;f5a5a857f5c5:32891 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765e5e72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:11,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:25:11,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:25:11,096 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:25:11,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:25:11,100 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:11,102 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846247, jitterRate=0.07605914771556854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:25:11,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300711076Initializing all the Stores at 1733300711080 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300711080Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300711080Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300711080Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300711080Cleaning up temporary data from old regions at 1733300711095 (+15 ms)Region opened successfully at 1733300711103 (+8 ms) 2024-12-04T08:25:11,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:25:11,104 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:25:11,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:25:11,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:25:11,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:25:11,106 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:11,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300711104Disabling compacts and flushes for region at 1733300711104Disabling writes for close at 1733300711104Writing region close event to WAL at 1733300711106 (+2 ms)Closed at 1733300711106 2024-12-04T08:25:11,109 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:11,109 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:25:11,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:25:11,113 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:25:11,115 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:25:11,115 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:32891 2024-12-04T08:25:11,116 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:25:11,116 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:25:11,116 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:25:11,117 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,43255,1733300710692 with port=32891, startcode=1733300710761 2024-12-04T08:25:11,117 DEBUG [RS:0;f5a5a857f5c5:32891 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:25:11,126 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35363, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:25:11,127 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43255 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,127 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43255 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,130 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d 2024-12-04T08:25:11,130 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45531 2024-12-04T08:25:11,130 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:25:11,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:11,134 DEBUG [RS:0;f5a5a857f5c5:32891 {}] zookeeper.ZKUtil(111): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,134 WARN [RS:0;f5a5a857f5c5:32891 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:11,134 INFO [RS:0;f5a5a857f5c5:32891 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:11,135 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/WALs/f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,135 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,32891,1733300710761] 2024-12-04T08:25:11,148 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:25:11,155 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:25:11,156 INFO [RS:0;f5a5a857f5c5:32891 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:25:11,156 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,160 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:25:11,161 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:25:11,162 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,162 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,162 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,162 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,162 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,162 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,162 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:11,163 DEBUG [RS:0;f5a5a857f5c5:32891 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:11,166 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,166 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,166 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,166 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,166 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,166 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,32891,1733300710761-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:11,189 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:25:11,189 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,32891,1733300710761-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,189 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,189 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.Replication(171): f5a5a857f5c5,32891,1733300710761 started 2024-12-04T08:25:11,214 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,214 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,32891,1733300710761, RpcServer on f5a5a857f5c5/172.17.0.2:32891, sessionid=0x1017c93d79f0001 2024-12-04T08:25:11,214 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:25:11,215 DEBUG [RS:0;f5a5a857f5c5:32891 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,215 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,32891,1733300710761' 2024-12-04T08:25:11,215 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:25:11,216 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:25:11,216 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:25:11,216 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:25:11,216 DEBUG [RS:0;f5a5a857f5c5:32891 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,216 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,32891,1733300710761' 2024-12-04T08:25:11,216 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:25:11,217 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:25:11,218 DEBUG [RS:0;f5a5a857f5c5:32891 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:25:11,218 INFO [RS:0;f5a5a857f5c5:32891 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:25:11,218 INFO [RS:0;f5a5a857f5c5:32891 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:25:11,265 WARN [f5a5a857f5c5:43255 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:25:11,321 INFO [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C32891%2C1733300710761, suffix=, logDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/WALs/f5a5a857f5c5,32891,1733300710761, archiveDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/oldWALs, maxLogs=32 2024-12-04T08:25:11,323 INFO [RS:0;f5a5a857f5c5:32891 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C32891%2C1733300710761.1733300711323 2024-12-04T08:25:11,331 INFO [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/WALs/f5a5a857f5c5,32891,1733300710761/f5a5a857f5c5%2C32891%2C1733300710761.1733300711323 2024-12-04T08:25:11,332 DEBUG [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34849:34849),(127.0.0.1/127.0.0.1:42635:42635)] 2024-12-04T08:25:11,516 DEBUG [f5a5a857f5c5:43255 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:25:11,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,518 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,32891,1733300710761, state=OPENING 2024-12-04T08:25:11,520 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:25:11,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,522 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:25:11,522 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:11,522 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:11,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,32891,1733300710761}] 2024-12-04T08:25:11,676 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:25:11,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43991, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:25:11,683 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:25:11,683 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:11,685 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C32891%2C1733300710761.meta, suffix=.meta, logDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/WALs/f5a5a857f5c5,32891,1733300710761, archiveDir=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/oldWALs, maxLogs=32 2024-12-04T08:25:11,687 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C32891%2C1733300710761.meta.1733300711687.meta 2024-12-04T08:25:11,693 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/WALs/f5a5a857f5c5,32891,1733300710761/f5a5a857f5c5%2C32891%2C1733300710761.meta.1733300711687.meta 2024-12-04T08:25:11,694 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34849:34849),(127.0.0.1/127.0.0.1:42635:42635)] 2024-12-04T08:25:11,695 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:11,695 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:25:11,695 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:25:11,695 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:25:11,696 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:25:11,696 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:11,696 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:25:11,696 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:25:11,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:25:11,698 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:25:11,699 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:25:11,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:25:11,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:25:11,702 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:25:11,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,703 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:25:11,703 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:25:11,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:11,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:11,704 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:25:11,705 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740 2024-12-04T08:25:11,706 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740 2024-12-04T08:25:11,708 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:25:11,708 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:25:11,709 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:25:11,710 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:25:11,711 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787131, jitterRate=8.891969919204712E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:25:11,711 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:25:11,712 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300711696Writing region info on filesystem at 1733300711696Initializing all the Stores at 1733300711697 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300711697Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300711697Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300711697Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300711697Cleaning up temporary data from old regions at 1733300711708 (+11 ms)Running coprocessor post-open hooks at 1733300711712 (+4 ms)Region opened successfully at 1733300711712 2024-12-04T08:25:11,714 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300711676 2024-12-04T08:25:11,717 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:25:11,717 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:25:11,718 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,719 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,32891,1733300710761, state=OPEN 2024-12-04T08:25:11,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:25:11,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:25:11,725 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:11,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:11,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:25:11,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,32891,1733300710761 in 203 msec 2024-12-04T08:25:11,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:25:11,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-12-04T08:25:11,733 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:11,733 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:25:11,734 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:25:11,735 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,32891,1733300710761, seqNum=-1] 2024-12-04T08:25:11,735 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:25:11,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43309, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:25:11,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 724 msec 2024-12-04T08:25:11,744 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300711744, completionTime=-1 2024-12-04T08:25:11,744 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:25:11,744 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:25:11,746 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:25:11,746 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300771746 2024-12-04T08:25:11,746 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733300831746 2024-12-04T08:25:11,746 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-04T08:25:11,747 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,43255,1733300710692-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,747 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,43255,1733300710692-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,747 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,43255,1733300710692-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,747 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:43255, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,747 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,747 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,749 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.941sec 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,43255,1733300710692-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:11,752 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,43255,1733300710692-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:25:11,755 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:25:11,756 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:25:11,756 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,43255,1733300710692-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:11,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5d9cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:11,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,43255,-1 for getting cluster id 2024-12-04T08:25:11,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:25:11,791 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e0ac2d2-6d59-44c5-8652-5f8ad1134681' 2024-12-04T08:25:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:25:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e0ac2d2-6d59-44c5-8652-5f8ad1134681" 2024-12-04T08:25:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300aa63e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,43255,-1] 2024-12-04T08:25:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:25:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:11,795 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:25:11,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf692f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:11,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:25:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,32891,1733300710761, seqNum=-1] 2024-12-04T08:25:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:25:11,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:25:11,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:11,803 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:11,807 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:25:11,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:25:11,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:25:11,807 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:11,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:11,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:11,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:25:11,808 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:25:11,808 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=358456280, stopped=false 2024-12-04T08:25:11,808 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,43255,1733300710692 2024-12-04T08:25:11,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:11,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:11,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:11,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:25:11,810 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:25:11,810 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:11,810 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:11,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:11,811 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,32891,1733300710761' ***** 2024-12-04T08:25:11,811 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:25:11,811 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:11,811 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:25:11,811 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:25:11,811 INFO [RS:0;f5a5a857f5c5:32891 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:25:11,811 INFO [RS:0;f5a5a857f5c5:32891 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:25:11,811 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:11,811 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:11,812 INFO [RS:0;f5a5a857f5c5:32891 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:32891. 2024-12-04T08:25:11,812 DEBUG [RS:0;f5a5a857f5c5:32891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:11,812 DEBUG [RS:0;f5a5a857f5c5:32891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:11,812 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:25:11,812 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:25:11,812 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:25:11,812 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:25:11,812 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T08:25:11,812 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T08:25:11,812 DEBUG [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T08:25:11,812 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:25:11,813 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:25:11,813 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:25:11,813 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:25:11,813 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:25:11,813 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-12-04T08:25:11,833 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/.tmp/ns/92bfd14f22a44648abb48e5dd0ca164e is 43, key is default/ns:d/1733300711737/Put/seqid=0 2024-12-04T08:25:11,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741835_1011 (size=5153) 2024-12-04T08:25:11,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741835_1011 (size=5153) 2024-12-04T08:25:11,840 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/.tmp/ns/92bfd14f22a44648abb48e5dd0ca164e 2024-12-04T08:25:11,849 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/.tmp/ns/92bfd14f22a44648abb48e5dd0ca164e as hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/ns/92bfd14f22a44648abb48e5dd0ca164e 2024-12-04T08:25:11,857 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/ns/92bfd14f22a44648abb48e5dd0ca164e, entries=2, sequenceid=6, filesize=5.0 K 2024-12-04T08:25:11,859 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 46ms, sequenceid=6, compaction requested=false 2024-12-04T08:25:11,859 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T08:25:11,865 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T08:25:11,865 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:25:11,865 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:11,866 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300711812Running coprocessor pre-close hooks at 1733300711812Disabling compacts and flushes for region at 1733300711812Disabling writes for close at 1733300711813 (+1 ms)Obtaining lock to block concurrent updates at 1733300711813Preparing flush snapshotting stores in 1588230740 at 1733300711813Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733300711813Flushing stores of hbase:meta,,1.1588230740 at 1733300711815 (+2 ms)Flushing 1588230740/ns: creating writer at 1733300711815Flushing 1588230740/ns: appending metadata at 1733300711832 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733300711832Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67309d28: reopening flushed file at 1733300711848 (+16 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 46ms, sequenceid=6, compaction requested=false at 1733300711859 (+11 ms)Writing region close event to WAL at 1733300711860 (+1 ms)Running coprocessor post-close hooks at 1733300711865 (+5 ms)Closed at 1733300711865 2024-12-04T08:25:11,866 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:12,013 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,32891,1733300710761; all regions closed. 2024-12-04T08:25:12,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741834_1010 (size=1152) 2024-12-04T08:25:12,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741834_1010 (size=1152) 2024-12-04T08:25:12,022 DEBUG [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/oldWALs 2024-12-04T08:25:12,022 INFO [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C32891%2C1733300710761.meta:.meta(num 1733300711687) 2024-12-04T08:25:12,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741833_1009 (size=93) 2024-12-04T08:25:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741833_1009 (size=93) 2024-12-04T08:25:12,030 DEBUG [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/oldWALs 2024-12-04T08:25:12,030 INFO [RS:0;f5a5a857f5c5:32891 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C32891%2C1733300710761:(num 1733300711323) 2024-12-04T08:25:12,030 DEBUG [RS:0;f5a5a857f5c5:32891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:12,030 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:12,030 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:12,030 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:12,031 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:12,031 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:12,031 INFO [RS:0;f5a5a857f5c5:32891 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32891 2024-12-04T08:25:12,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:12,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,32891,1733300710761 2024-12-04T08:25:12,034 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:12,034 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,32891,1733300710761] 2024-12-04T08:25:12,037 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,32891,1733300710761 already deleted, retry=false 2024-12-04T08:25:12,037 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,32891,1733300710761 expired; onlineServers=0 2024-12-04T08:25:12,037 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,43255,1733300710692' ***** 2024-12-04T08:25:12,038 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:25:12,038 INFO [M:0;f5a5a857f5c5:43255 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:12,038 INFO [M:0;f5a5a857f5c5:43255 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:12,038 DEBUG [M:0;f5a5a857f5c5:43255 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:25:12,038 DEBUG [M:0;f5a5a857f5c5:43255 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:25:12,038 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:25:12,038 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300711041 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300711041,5,FailOnTimeoutGroup] 2024-12-04T08:25:12,038 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300711042 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300711042,5,FailOnTimeoutGroup] 2024-12-04T08:25:12,038 INFO [M:0;f5a5a857f5c5:43255 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:12,038 INFO [M:0;f5a5a857f5c5:43255 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:12,038 DEBUG [M:0;f5a5a857f5c5:43255 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:25:12,038 INFO [M:0;f5a5a857f5c5:43255 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:25:12,039 INFO [M:0;f5a5a857f5c5:43255 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:25:12,039 INFO [M:0;f5a5a857f5c5:43255 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:25:12,039 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:25:12,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:25:12,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:12,041 DEBUG [M:0;f5a5a857f5c5:43255 {}] zookeeper.ZKUtil(347): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:25:12,041 WARN [M:0;f5a5a857f5c5:43255 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:25:12,041 INFO [M:0;f5a5a857f5c5:43255 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/.lastflushedseqids 2024-12-04T08:25:12,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741836_1012 (size=99) 2024-12-04T08:25:12,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741836_1012 (size=99) 2024-12-04T08:25:12,054 INFO [M:0;f5a5a857f5c5:43255 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:25:12,054 INFO [M:0;f5a5a857f5c5:43255 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:25:12,054 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:25:12,054 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:12,054 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:12,055 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:25:12,055 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:12,055 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-12-04T08:25:12,076 DEBUG [M:0;f5a5a857f5c5:43255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02b7c233ef324b65ae048d453409d645 is 82, key is hbase:meta,,1/info:regioninfo/1733300711718/Put/seqid=0 2024-12-04T08:25:12,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741837_1013 (size=5672) 2024-12-04T08:25:12,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741837_1013 (size=5672) 2024-12-04T08:25:12,084 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02b7c233ef324b65ae048d453409d645 2024-12-04T08:25:12,115 DEBUG [M:0;f5a5a857f5c5:43255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/817935d32b1a4f338e0d835d6c2d22ec is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733300711743/Put/seqid=0 2024-12-04T08:25:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741838_1014 (size=5275) 2024-12-04T08:25:12,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741838_1014 (size=5275) 2024-12-04T08:25:12,123 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/817935d32b1a4f338e0d835d6c2d22ec 2024-12-04T08:25:12,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:12,136 INFO [RS:0;f5a5a857f5c5:32891 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:12,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32891-0x1017c93d79f0001, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:12,136 INFO [RS:0;f5a5a857f5c5:32891 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,32891,1733300710761; zookeeper connection closed. 2024-12-04T08:25:12,136 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c7cbd55 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c7cbd55 2024-12-04T08:25:12,137 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T08:25:12,147 DEBUG [M:0;f5a5a857f5c5:43255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/707a3b49469c46b3a63669695ff2ca37 is 69, key is f5a5a857f5c5,32891,1733300710761/rs:state/1733300711127/Put/seqid=0 2024-12-04T08:25:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741839_1015 (size=5156) 2024-12-04T08:25:12,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741839_1015 (size=5156) 2024-12-04T08:25:12,155 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/707a3b49469c46b3a63669695ff2ca37 2024-12-04T08:25:12,178 DEBUG [M:0;f5a5a857f5c5:43255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1b794d15dcf04b228b49fd723b01c3da is 52, key is load_balancer_on/state:d/1733300711805/Put/seqid=0 2024-12-04T08:25:12,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741840_1016 (size=5056) 2024-12-04T08:25:12,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741840_1016 (size=5056) 2024-12-04T08:25:12,185 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1b794d15dcf04b228b49fd723b01c3da 2024-12-04T08:25:12,194 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02b7c233ef324b65ae048d453409d645 as hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/02b7c233ef324b65ae048d453409d645 2024-12-04T08:25:12,202 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/02b7c233ef324b65ae048d453409d645, entries=8, sequenceid=29, filesize=5.5 K 2024-12-04T08:25:12,204 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/817935d32b1a4f338e0d835d6c2d22ec as hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/817935d32b1a4f338e0d835d6c2d22ec 2024-12-04T08:25:12,211 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/817935d32b1a4f338e0d835d6c2d22ec, entries=3, sequenceid=29, filesize=5.2 K 2024-12-04T08:25:12,212 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/707a3b49469c46b3a63669695ff2ca37 as hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/707a3b49469c46b3a63669695ff2ca37 2024-12-04T08:25:12,219 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/707a3b49469c46b3a63669695ff2ca37, entries=1, sequenceid=29, filesize=5.0 K 2024-12-04T08:25:12,221 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1b794d15dcf04b228b49fd723b01c3da as hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1b794d15dcf04b228b49fd723b01c3da 2024-12-04T08:25:12,228 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45531/user/jenkins/test-data/7fdc0aad-8542-7ae0-7533-fc66c36d4d9d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1b794d15dcf04b228b49fd723b01c3da, entries=1, sequenceid=29, filesize=4.9 K 2024-12-04T08:25:12,229 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 174ms, sequenceid=29, compaction requested=false 2024-12-04T08:25:12,233 INFO [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:12,234 DEBUG [M:0;f5a5a857f5c5:43255 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300712054Disabling compacts and flushes for region at 1733300712054Disabling writes for close at 1733300712055 (+1 ms)Obtaining lock to block concurrent updates at 1733300712055Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300712055Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733300712055Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300712056 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300712056Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300712075 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300712075Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300712092 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300712115 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300712115Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300712129 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300712147 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300712147Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300712162 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300712178 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300712178Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c26d053: reopening flushed file at 1733300712193 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d6f56d2: reopening flushed file at 1733300712203 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6de064fb: reopening flushed file at 1733300712211 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@770164ef: reopening flushed file at 1733300712219 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 174ms, sequenceid=29, compaction requested=false at 1733300712230 (+11 ms)Writing region close event to WAL at 1733300712233 (+3 ms)Closed at 1733300712233 2024-12-04T08:25:12,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:12,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45807 is added to blk_1073741830_1006 (size=10311) 2024-12-04T08:25:12,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741830_1006 (size=10311) 2024-12-04T08:25:12,240 INFO [M:0;f5a5a857f5c5:43255 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:25:12,240 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:12,240 INFO [M:0;f5a5a857f5c5:43255 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43255 2024-12-04T08:25:12,241 INFO [M:0;f5a5a857f5c5:43255 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:12,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:12,342 INFO [M:0;f5a5a857f5c5:43255 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:12,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43255-0x1017c93d79f0000, quorum=127.0.0.1:49184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:12,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38d766c7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:12,346 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@12686d25{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:12,346 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:12,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ab06e68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:12,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c30f553{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:12,348 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:12,348 WARN [BP-827450206-172.17.0.2-1733300709842 heartbeating to localhost/127.0.0.1:45531 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:12,348 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:12,348 WARN [BP-827450206-172.17.0.2-1733300709842 heartbeating to localhost/127.0.0.1:45531 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-827450206-172.17.0.2-1733300709842 (Datanode Uuid 92796feb-b164-4241-a1d3-4551808dd045) service to localhost/127.0.0.1:45531 2024-12-04T08:25:12,349 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data3/current/BP-827450206-172.17.0.2-1733300709842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:12,349 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data4/current/BP-827450206-172.17.0.2-1733300709842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:12,350 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:12,353 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@78fa6004{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:12,353 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6b5fac92{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:12,354 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:12,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4620cd8a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:12,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3270c9ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:12,355 WARN [BP-827450206-172.17.0.2-1733300709842 heartbeating to localhost/127.0.0.1:45531 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:12,355 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:12,356 WARN [BP-827450206-172.17.0.2-1733300709842 heartbeating to localhost/127.0.0.1:45531 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-827450206-172.17.0.2-1733300709842 (Datanode Uuid d9bd77ee-49d0-4ed3-b636-58e05a737b53) service to localhost/127.0.0.1:45531 2024-12-04T08:25:12,356 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:12,356 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data1/current/BP-827450206-172.17.0.2-1733300709842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:12,356 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/cluster_57079c50-b7e7-a2bf-aeb2-cd4d25db20c6/data/data2/current/BP-827450206-172.17.0.2-1733300709842 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:12,357 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:12,362 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bc50e3b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:25:12,363 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8c19991{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:12,363 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:12,363 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d9de743{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:12,363 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@630e1a46{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:12,370 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:25:12,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:25:12,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.log.dir so I do NOT create it in target/test-data/a53da31c-579c-6961-1370-b92276dd02aa 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/76eed0fe-9a91-350c-c920-11c7a217cc2a/hadoop.tmp.dir so I do NOT create it in target/test-data/a53da31c-579c-6961-1370-b92276dd02aa 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7, deleteOnExit=true 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/test.cache.data in system properties and HBase conf 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:25:12,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:25:12,391 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:25:12,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:25:12,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:25:12,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:25:12,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:25:12,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:25:12,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:25:12,407 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:25:12,501 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:12,511 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:12,521 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:12,521 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:12,521 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:25:12,522 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:12,525 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d8a9c69{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:12,526 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6abea1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:12,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6469263a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-38391-hadoop-hdfs-3_4_1-tests_jar-_-any-8850165831146912518/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:25:12,670 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2324be38{HTTP/1.1, (http/1.1)}{localhost:38391} 2024-12-04T08:25:12,670 INFO [Time-limited test {}] server.Server(415): Started @104084ms 2024-12-04T08:25:12,690 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:25:12,775 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:12,778 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:12,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:12,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:12,779 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:25:12,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@259c861e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:12,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2df55a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:12,900 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d790455{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-38039-hadoop-hdfs-3_4_1-tests_jar-_-any-2293549949644919209/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:12,901 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@795c5052{HTTP/1.1, (http/1.1)}{localhost:38039} 2024-12-04T08:25:12,901 INFO [Time-limited test {}] server.Server(415): Started @104315ms 2024-12-04T08:25:12,903 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:12,935 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:12,938 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:12,940 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:12,940 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:12,940 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:12,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fe8f7b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:12,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@520d17ab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:13,010 WARN [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data1/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:13,010 WARN [Thread-659 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data2/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:13,031 WARN [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c88541cc41bc597 with lease ID 0x4beb241b305ac8eb: Processing first storage report for DS-ca06bcae-5faf-4549-b4ad-e902365b392d from datanode DatanodeRegistration(127.0.0.1:44007, datanodeUuid=dcc214a0-f297-48df-83a9-7fb8417285b3, infoPort=45619, infoSecurePort=0, ipcPort=45357, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c88541cc41bc597 with lease ID 0x4beb241b305ac8eb: from storage DS-ca06bcae-5faf-4549-b4ad-e902365b392d node DatanodeRegistration(127.0.0.1:44007, datanodeUuid=dcc214a0-f297-48df-83a9-7fb8417285b3, infoPort=45619, infoSecurePort=0, ipcPort=45357, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c88541cc41bc597 with lease ID 0x4beb241b305ac8eb: Processing first storage report for DS-a03282d5-4c26-49b7-84fc-4ae71c039480 from datanode DatanodeRegistration(127.0.0.1:44007, datanodeUuid=dcc214a0-f297-48df-83a9-7fb8417285b3, infoPort=45619, infoSecurePort=0, ipcPort=45357, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c88541cc41bc597 with lease ID 0x4beb241b305ac8eb: from storage DS-a03282d5-4c26-49b7-84fc-4ae71c039480 node DatanodeRegistration(127.0.0.1:44007, datanodeUuid=dcc214a0-f297-48df-83a9-7fb8417285b3, infoPort=45619, infoSecurePort=0, ipcPort=45357, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:13,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@85c422e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-35331-hadoop-hdfs-3_4_1-tests_jar-_-any-14912104763515818463/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:13,064 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@293f42b5{HTTP/1.1, (http/1.1)}{localhost:35331} 2024-12-04T08:25:13,064 INFO [Time-limited test {}] server.Server(415): Started @104478ms 2024-12-04T08:25:13,066 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:13,167 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:13,172 WARN [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data3/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:13,172 WARN [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data4/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:13,196 WARN [Thread-673 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdcde1e82b4d96059 with lease ID 0x4beb241b305ac8ec: Processing first storage report for DS-fb1b7e16-a807-4b59-88c3-8909635b5810 from datanode DatanodeRegistration(127.0.0.1:35749, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=43615, infoSecurePort=0, ipcPort=36189, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdcde1e82b4d96059 with lease ID 0x4beb241b305ac8ec: from storage DS-fb1b7e16-a807-4b59-88c3-8909635b5810 node DatanodeRegistration(127.0.0.1:35749, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=43615, infoSecurePort=0, ipcPort=36189, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdcde1e82b4d96059 with lease ID 0x4beb241b305ac8ec: Processing first storage report for DS-35456c5a-541d-4df1-8d1a-a1e0d1aa41f4 from datanode DatanodeRegistration(127.0.0.1:35749, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=43615, infoSecurePort=0, ipcPort=36189, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdcde1e82b4d96059 with lease ID 0x4beb241b305ac8ec: from storage DS-35456c5a-541d-4df1-8d1a-a1e0d1aa41f4 node DatanodeRegistration(127.0.0.1:35749, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=43615, infoSecurePort=0, ipcPort=36189, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:13,302 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa 2024-12-04T08:25:13,307 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/zookeeper_0, clientPort=49602, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:25:13,308 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49602 2024-12-04T08:25:13,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,311 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:25:13,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:25:13,324 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94 with version=8 2024-12-04T08:25:13,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:25:13,327 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:25:13,327 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:13,328 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41417 2024-12-04T08:25:13,329 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41417 connecting to ZooKeeper ensemble=127.0.0.1:49602 2024-12-04T08:25:13,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414170x0, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:13,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41417-0x1017c93e1e80000 connected 2024-12-04T08:25:13,361 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:13,366 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94, hbase.cluster.distributed=false 2024-12-04T08:25:13,368 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:13,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41417 2024-12-04T08:25:13,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41417 2024-12-04T08:25:13,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41417 2024-12-04T08:25:13,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41417 2024-12-04T08:25:13,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41417 2024-12-04T08:25:13,386 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:13,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:13,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:13,386 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:13,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:13,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:13,386 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:25:13,387 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:13,387 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37469 2024-12-04T08:25:13,389 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37469 connecting to ZooKeeper ensemble=127.0.0.1:49602 2024-12-04T08:25:13,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,391 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374690x0, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:13,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37469-0x1017c93e1e80001 connected 2024-12-04T08:25:13,397 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:13,397 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:25:13,398 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:25:13,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:25:13,400 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:13,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37469 2024-12-04T08:25:13,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37469 2024-12-04T08:25:13,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37469 2024-12-04T08:25:13,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37469 2024-12-04T08:25:13,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37469 2024-12-04T08:25:13,417 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:41417 2024-12-04T08:25:13,418 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:13,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:13,420 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:25:13,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,423 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:25:13,423 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,41417,1733300713326 from backup master directory 2024-12-04T08:25:13,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,426 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:13,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:13,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:13,432 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/hbase.id] with ID: 08bc6629-3930-48a6-80b3-6222905b4803 2024-12-04T08:25:13,432 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/.tmp/hbase.id 2024-12-04T08:25:13,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:25:13,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:25:13,439 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/.tmp/hbase.id]:[hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/hbase.id] 2024-12-04T08:25:13,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:13,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:25:13,455 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T08:25:13,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:13,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:13,466 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:25:13,467 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:25:13,467 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:13,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:25:13,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:25:13,479 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store 2024-12-04T08:25:13,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:13,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:13,488 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:13,488 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:25:13,488 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:13,488 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:13,488 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:25:13,488 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:13,488 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:13,488 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300713488Disabling compacts and flushes for region at 1733300713488Disabling writes for close at 1733300713488Writing region close event to WAL at 1733300713488Closed at 1733300713488 2024-12-04T08:25:13,489 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/.initializing 2024-12-04T08:25:13,489 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,492 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C41417%2C1733300713326, suffix=, logDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326, archiveDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/oldWALs, maxLogs=10 2024-12-04T08:25:13,493 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 2024-12-04T08:25:13,504 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 2024-12-04T08:25:13,509 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45619:45619),(127.0.0.1/127.0.0.1:43615:43615)] 2024-12-04T08:25:13,515 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:13,516 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:13,516 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,516 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:25:13,523 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:13,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:25:13,526 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:13,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:25:13,528 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:13,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:25:13,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:13,531 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,532 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,533 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,535 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,535 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,535 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:25:13,537 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:13,541 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:13,542 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846518, jitterRate=0.0764039158821106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:25:13,543 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300713516Initializing all the Stores at 1733300713518 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300713518Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300713520 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300713520Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300713520Cleaning up temporary data from old regions at 1733300713535 (+15 ms)Region opened successfully at 1733300713543 (+8 ms) 2024-12-04T08:25:13,543 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:25:13,548 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6b12b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:13,550 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:25:13,550 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:25:13,550 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:25:13,550 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:25:13,551 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T08:25:13,551 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T08:25:13,551 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:25:13,555 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:25:13,556 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:25:13,557 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:25:13,558 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:25:13,558 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:25:13,560 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:25:13,561 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:25:13,562 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:25:13,565 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:25:13,566 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:25:13,567 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:25:13,570 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:25:13,571 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:25:13,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:13,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:13,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,575 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,41417,1733300713326, sessionid=0x1017c93e1e80000, setting cluster-up flag (Was=false) 2024-12-04T08:25:13,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,586 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:25:13,587 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:13,602 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:25:13,603 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:13,608 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:25:13,610 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:13,611 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:25:13,611 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:25:13,611 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,41417,1733300713326 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:13,613 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,615 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:13,616 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:25:13,617 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,617 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:25:13,653 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300743653 2024-12-04T08:25:13,654 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:25:13,654 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:25:13,654 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:25:13,654 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:25:13,654 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:25:13,654 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:25:13,655 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,656 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:25:13,656 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:25:13,656 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:25:13,656 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:25:13,656 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:25:13,657 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300713656,5,FailOnTimeoutGroup] 2024-12-04T08:25:13,660 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300713657,5,FailOnTimeoutGroup] 2024-12-04T08:25:13,660 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,660 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:25:13,660 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,660 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:13,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:13,671 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:25:13,671 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94 2024-12-04T08:25:13,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:25:13,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:25:13,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:13,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:25:13,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:25:13,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:13,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:25:13,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:25:13,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:13,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:25:13,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:25:13,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:13,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:25:13,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:25:13,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:13,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:13,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:25:13,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740 2024-12-04T08:25:13,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740 2024-12-04T08:25:13,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:25:13,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:25:13,695 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:25:13,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:25:13,699 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:13,700 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790469, jitterRate=0.005133435130119324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:25:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300713679Initializing all the Stores at 1733300713679Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300713679Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300713680 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300713680Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300713680Cleaning up temporary data from old regions at 1733300713694 (+14 ms)Region opened successfully at 1733300713700 (+6 ms) 2024-12-04T08:25:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:25:13,701 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:25:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:25:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:25:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:25:13,701 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:13,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300713701Disabling compacts and flushes for region at 1733300713701Disabling writes for close at 1733300713701Writing region close event to WAL at 1733300713701Closed at 1733300713701 2024-12-04T08:25:13,703 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:13,703 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:25:13,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:25:13,705 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:25:13,707 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:25:13,711 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(746): ClusterId : 08bc6629-3930-48a6-80b3-6222905b4803 2024-12-04T08:25:13,711 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:25:13,713 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:25:13,713 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:25:13,716 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:25:13,718 DEBUG [RS:0;f5a5a857f5c5:37469 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45fb486e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:13,734 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:37469 2024-12-04T08:25:13,734 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:25:13,734 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:25:13,734 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:25:13,735 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,41417,1733300713326 with port=37469, startcode=1733300713386 2024-12-04T08:25:13,735 DEBUG [RS:0;f5a5a857f5c5:37469 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:25:13,739 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44699, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:25:13,740 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41417 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:13,740 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41417 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:13,742 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94 2024-12-04T08:25:13,742 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36333 2024-12-04T08:25:13,742 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:25:13,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:13,745 DEBUG [RS:0;f5a5a857f5c5:37469 {}] zookeeper.ZKUtil(111): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:13,745 WARN [RS:0;f5a5a857f5c5:37469 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:13,745 INFO [RS:0;f5a5a857f5c5:37469 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:13,745 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:13,752 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,37469,1733300713386] 2024-12-04T08:25:13,754 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:25:13,756 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:25:13,757 INFO [RS:0;f5a5a857f5c5:37469 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:25:13,757 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,760 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:25:13,761 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:25:13,761 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,761 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,761 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:13,762 DEBUG [RS:0;f5a5a857f5c5:37469 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:13,764 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,764 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,764 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,764 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,764 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,764 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,37469,1733300713386-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:13,782 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:25:13,782 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,37469,1733300713386-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,782 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,782 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.Replication(171): f5a5a857f5c5,37469,1733300713386 started 2024-12-04T08:25:13,799 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:13,799 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,37469,1733300713386, RpcServer on f5a5a857f5c5/172.17.0.2:37469, sessionid=0x1017c93e1e80001 2024-12-04T08:25:13,800 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:25:13,800 DEBUG [RS:0;f5a5a857f5c5:37469 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:13,800 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,37469,1733300713386' 2024-12-04T08:25:13,800 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:25:13,801 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:25:13,801 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:25:13,801 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:25:13,801 DEBUG [RS:0;f5a5a857f5c5:37469 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:13,801 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,37469,1733300713386' 2024-12-04T08:25:13,801 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:25:13,802 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:25:13,803 DEBUG [RS:0;f5a5a857f5c5:37469 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:25:13,803 INFO [RS:0;f5a5a857f5c5:37469 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:25:13,803 INFO [RS:0;f5a5a857f5c5:37469 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:25:13,857 WARN [f5a5a857f5c5:41417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:25:13,906 INFO [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C37469%2C1733300713386, suffix=, logDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386, archiveDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs, maxLogs=32 2024-12-04T08:25:13,907 INFO [RS:0;f5a5a857f5c5:37469 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 2024-12-04T08:25:13,914 INFO [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 2024-12-04T08:25:13,919 DEBUG [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43615:43615),(127.0.0.1/127.0.0.1:45619:45619)] 2024-12-04T08:25:13,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:25:13,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T08:25:13,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-04T08:25:14,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:14,028 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:14,108 DEBUG [f5a5a857f5c5:41417 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:25:14,108 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:14,110 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,37469,1733300713386, state=OPENING 2024-12-04T08:25:14,112 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:25:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:14,114 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:25:14,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:14,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:14,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,37469,1733300713386}] 2024-12-04T08:25:14,268 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:25:14,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47365, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:25:14,274 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:25:14,274 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:14,277 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C37469%2C1733300713386.meta, suffix=.meta, logDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386, archiveDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs, maxLogs=32 2024-12-04T08:25:14,278 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta 2024-12-04T08:25:14,286 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta 2024-12-04T08:25:14,288 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43615:43615),(127.0.0.1/127.0.0.1:45619:45619)] 2024-12-04T08:25:14,290 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:14,290 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:25:14,290 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:25:14,291 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:25:14,291 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:25:14,291 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:14,291 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:25:14,291 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:25:14,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:25:14,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:25:14,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:14,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:14,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:25:14,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:25:14,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:14,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:14,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:25:14,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:25:14,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:14,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:14,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:25:14,303 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:25:14,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:14,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:14,304 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:25:14,304 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740 2024-12-04T08:25:14,306 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740 2024-12-04T08:25:14,307 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:25:14,307 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:25:14,308 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:25:14,310 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:25:14,311 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=856200, jitterRate=0.08871541917324066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:25:14,311 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:25:14,311 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300714291Writing region info on filesystem at 1733300714291Initializing all the Stores at 1733300714292 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300714293 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300714296 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300714296Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300714296Cleaning up temporary data from old regions at 1733300714307 (+11 ms)Running coprocessor post-open hooks at 1733300714311 (+4 ms)Region opened successfully at 1733300714311 2024-12-04T08:25:14,313 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300714267 2024-12-04T08:25:14,317 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:25:14,317 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:25:14,317 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:14,319 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,37469,1733300713386, state=OPEN 2024-12-04T08:25:14,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:25:14,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:25:14,326 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:14,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:14,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:14,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:25:14,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,37469,1733300713386 in 212 msec 2024-12-04T08:25:14,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:25:14,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 626 msec 2024-12-04T08:25:14,334 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:14,334 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:25:14,336 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:25:14,336 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,37469,1733300713386, seqNum=-1] 2024-12-04T08:25:14,336 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:25:14,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47229, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:25:14,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 733 msec 2024-12-04T08:25:14,344 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300714344, completionTime=-1 2024-12-04T08:25:14,344 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:25:14,344 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300774347 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733300834347 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,41417,1733300713326-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,41417,1733300713326-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,347 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,41417,1733300713326-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,348 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:41417, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,348 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,348 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,350 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:25:14,352 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.926sec 2024-12-04T08:25:14,352 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:25:14,352 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:25:14,352 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:25:14,352 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:25:14,353 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:25:14,353 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,41417,1733300713326-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:14,353 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,41417,1733300713326-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:25:14,356 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:25:14,356 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:25:14,356 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,41417,1733300713326-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abbe752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:14,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,41417,-1 for getting cluster id 2024-12-04T08:25:14,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:25:14,409 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08bc6629-3930-48a6-80b3-6222905b4803' 2024-12-04T08:25:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:25:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08bc6629-3930-48a6-80b3-6222905b4803" 2024-12-04T08:25:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1535ec62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,41417,-1] 2024-12-04T08:25:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:25:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:14,412 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:25:14,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e8c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:14,413 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:25:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,37469,1733300713386, seqNum=-1] 2024-12-04T08:25:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:25:14,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:25:14,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:14,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:14,421 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:25:14,445 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:14,445 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:14,446 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:25:14,446 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:14,447 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35147 2024-12-04T08:25:14,448 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35147 connecting to ZooKeeper ensemble=127.0.0.1:49602 2024-12-04T08:25:14,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:14,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:14,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351470x0, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:14,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:351470x0, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-04T08:25:14,457 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-04T08:25:14,458 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35147-0x1017c93e1e80002 connected 2024-12-04T08:25:14,460 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:25:14,461 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:25:14,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:25:14,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:14,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35147 2024-12-04T08:25:14,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35147 2024-12-04T08:25:14,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35147 2024-12-04T08:25:14,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35147 2024-12-04T08:25:14,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35147 2024-12-04T08:25:14,473 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(746): ClusterId : 08bc6629-3930-48a6-80b3-6222905b4803 2024-12-04T08:25:14,473 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:25:14,475 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:25:14,475 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:25:14,477 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:25:14,478 DEBUG [RS:1;f5a5a857f5c5:35147 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7257e1ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:14,489 DEBUG [RS:1;f5a5a857f5c5:35147 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f5a5a857f5c5:35147 2024-12-04T08:25:14,489 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:25:14,489 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:25:14,489 DEBUG [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:25:14,490 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,41417,1733300713326 with port=35147, startcode=1733300714444 2024-12-04T08:25:14,490 DEBUG [RS:1;f5a5a857f5c5:35147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:25:14,492 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57657, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:25:14,492 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41417 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:14,492 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41417 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:14,494 DEBUG [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94 2024-12-04T08:25:14,494 DEBUG [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36333 2024-12-04T08:25:14,494 DEBUG [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:25:14,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:14,498 DEBUG [RS:1;f5a5a857f5c5:35147 {}] zookeeper.ZKUtil(111): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:14,498 WARN [RS:1;f5a5a857f5c5:35147 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:14,498 INFO [RS:1;f5a5a857f5c5:35147 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:14,498 DEBUG [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:14,498 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,35147,1733300714444] 2024-12-04T08:25:14,502 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:25:14,504 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:25:14,504 INFO [RS:1;f5a5a857f5c5:35147 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:25:14,504 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,504 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:25:14,505 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:25:14,505 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,506 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,507 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:14,507 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:14,507 DEBUG [RS:1;f5a5a857f5c5:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:14,507 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,507 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,507 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,507 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,507 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,507 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,35147,1733300714444-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:14,522 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:25:14,522 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,35147,1733300714444-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,522 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,522 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.Replication(171): f5a5a857f5c5,35147,1733300714444 started 2024-12-04T08:25:14,536 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:14,536 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,35147,1733300714444, RpcServer on f5a5a857f5c5/172.17.0.2:35147, sessionid=0x1017c93e1e80002 2024-12-04T08:25:14,536 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:25:14,537 DEBUG [RS:1;f5a5a857f5c5:35147 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:14,537 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,35147,1733300714444' 2024-12-04T08:25:14,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;f5a5a857f5c5:35147,5,FailOnTimeoutGroup] 2024-12-04T08:25:14,537 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:25:14,537 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-12-04T08:25:14,537 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:25:14,537 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T08:25:14,537 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:25:14,537 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:25:14,538 DEBUG [RS:1;f5a5a857f5c5:35147 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:14,538 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,35147,1733300714444' 2024-12-04T08:25:14,538 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:25:14,538 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:25:14,538 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:25:14,538 DEBUG [RS:1;f5a5a857f5c5:35147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:25:14,538 INFO [RS:1;f5a5a857f5c5:35147 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:25:14,538 INFO [RS:1;f5a5a857f5c5:35147 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:25:14,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fa8e196 2024-12-04T08:25:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T08:25:14,541 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T08:25:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T08:25:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T08:25:14,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T08:25:14,545 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:14,545 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T08:25:14,545 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:14,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-12-04T08:25:14,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T08:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:25:14,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741835_1011 (size=393) 2024-12-04T08:25:14,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741835_1011 (size=393) 2024-12-04T08:25:14,560 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:14,561 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3af4b7ac06a411c8ccf86f339fcc9ac1, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94 2024-12-04T08:25:14,562 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:14,563 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:14,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44007 is added to blk_1073741836_1012 (size=76) 2024-12-04T08:25:14,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35749 is added to blk_1073741836_1012 (size=76) 2024-12-04T08:25:14,572 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:14,572 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 3af4b7ac06a411c8ccf86f339fcc9ac1, disabling compactions & flushes 2024-12-04T08:25:14,572 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,572 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,572 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. after waiting 0 ms 2024-12-04T08:25:14,572 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,572 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,572 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: Waiting for close lock at 1733300714572Disabling compacts and flushes for region at 1733300714572Disabling writes for close at 1733300714572Writing region close event to WAL at 1733300714572Closed at 1733300714572 2024-12-04T08:25:14,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T08:25:14,574 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733300714574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300714574"}]},"ts":"1733300714574"} 2024-12-04T08:25:14,577 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T08:25:14,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T08:25:14,578 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300714578"}]},"ts":"1733300714578"} 2024-12-04T08:25:14,581 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-12-04T08:25:14,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3af4b7ac06a411c8ccf86f339fcc9ac1, ASSIGN}] 2024-12-04T08:25:14,582 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3af4b7ac06a411c8ccf86f339fcc9ac1, ASSIGN 2024-12-04T08:25:14,584 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3af4b7ac06a411c8ccf86f339fcc9ac1, ASSIGN; state=OFFLINE, location=f5a5a857f5c5,37469,1733300713386; forceNewPlan=false, retain=false 2024-12-04T08:25:14,641 INFO [RS:1;f5a5a857f5c5:35147 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C35147%2C1733300714444, suffix=, logDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444, archiveDir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs, maxLogs=32 2024-12-04T08:25:14,642 INFO [RS:1;f5a5a857f5c5:35147 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 2024-12-04T08:25:14,648 INFO [RS:1;f5a5a857f5c5:35147 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 2024-12-04T08:25:14,649 DEBUG [RS:1;f5a5a857f5c5:35147 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43615:43615),(127.0.0.1/127.0.0.1:45619:45619)] 2024-12-04T08:25:14,734 INFO [f5a5a857f5c5:41417 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T08:25:14,735 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3af4b7ac06a411c8ccf86f339fcc9ac1, regionState=OPENING, regionLocation=f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:14,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3af4b7ac06a411c8ccf86f339fcc9ac1, ASSIGN because future has completed 2024-12-04T08:25:14,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3af4b7ac06a411c8ccf86f339fcc9ac1, server=f5a5a857f5c5,37469,1733300713386}] 2024-12-04T08:25:14,896 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,897 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3af4b7ac06a411c8ccf86f339fcc9ac1, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:14,897 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,897 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:14,898 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,898 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,899 INFO [StoreOpener-3af4b7ac06a411c8ccf86f339fcc9ac1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,901 INFO [StoreOpener-3af4b7ac06a411c8ccf86f339fcc9ac1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3af4b7ac06a411c8ccf86f339fcc9ac1 columnFamilyName info 2024-12-04T08:25:14,901 DEBUG [StoreOpener-3af4b7ac06a411c8ccf86f339fcc9ac1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:14,902 INFO [StoreOpener-3af4b7ac06a411c8ccf86f339fcc9ac1-1 {}] regionserver.HStore(327): Store=3af4b7ac06a411c8ccf86f339fcc9ac1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:14,902 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,903 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,903 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,904 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,904 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,905 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,907 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:14,907 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3af4b7ac06a411c8ccf86f339fcc9ac1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755296, jitterRate=-0.039592042565345764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:25:14,908 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:14,908 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: Running coprocessor pre-open hook at 1733300714898Writing region info on filesystem at 1733300714898Initializing all the Stores at 1733300714899 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300714899Cleaning up temporary data from old regions at 1733300714904 (+5 ms)Running coprocessor post-open hooks at 1733300714908 (+4 ms)Region opened successfully at 1733300714908 2024-12-04T08:25:14,909 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1., pid=6, masterSystemTime=1733300714891 2024-12-04T08:25:14,912 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,912 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:14,913 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3af4b7ac06a411c8ccf86f339fcc9ac1, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:14,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3af4b7ac06a411c8ccf86f339fcc9ac1, server=f5a5a857f5c5,37469,1733300713386 because future has completed 2024-12-04T08:25:14,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T08:25:14,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3af4b7ac06a411c8ccf86f339fcc9ac1, server=f5a5a857f5c5,37469,1733300713386 in 179 msec 2024-12-04T08:25:14,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T08:25:14,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3af4b7ac06a411c8ccf86f339fcc9ac1, ASSIGN in 338 msec 2024-12-04T08:25:14,923 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T08:25:14,923 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300714923"}]},"ts":"1733300714923"} 2024-12-04T08:25:14,925 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-12-04T08:25:14,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T08:25:14,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 385 msec 2024-12-04T08:25:19,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:25:19,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:19,462 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:19,465 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:19,466 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:19,754 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-12-04T08:25:23,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T08:25:23,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T08:25:23,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T08:25:23,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-12-04T08:25:23,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:25:23,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T08:25:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:25:24,631 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-12-04T08:25:24,631 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-12-04T08:25:24,634 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T08:25:24,634 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:24,647 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:24,651 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:24,652 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:24,652 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:24,652 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:24,652 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@132d95f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:24,653 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fa27241{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:24,768 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@539abed5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-35999-hadoop-hdfs-3_4_1-tests_jar-_-any-10330473530036308944/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:24,768 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2046b984{HTTP/1.1, (http/1.1)}{localhost:35999} 2024-12-04T08:25:24,769 INFO [Time-limited test {}] server.Server(415): Started @116182ms 2024-12-04T08:25:24,770 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:24,810 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:24,813 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:24,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:24,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:24,814 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:24,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3986ff43{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:24,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c590722{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:24,869 WARN [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data5/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:24,869 WARN [Thread-831 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data6/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:24,886 WARN [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb16f01176ddfed33 with lease ID 0x4beb241b305ac8ed: Processing first storage report for DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d from datanode DatanodeRegistration(127.0.0.1:35991, datanodeUuid=03545c32-c536-4081-8ede-49e42a2abaff, infoPort=45529, infoSecurePort=0, ipcPort=40211, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb16f01176ddfed33 with lease ID 0x4beb241b305ac8ed: from storage DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d node DatanodeRegistration(127.0.0.1:35991, datanodeUuid=03545c32-c536-4081-8ede-49e42a2abaff, infoPort=45529, infoSecurePort=0, ipcPort=40211, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb16f01176ddfed33 with lease ID 0x4beb241b305ac8ed: Processing first storage report for DS-3d8b146c-4d34-4c7d-937b-e6c1a2a385cf from datanode DatanodeRegistration(127.0.0.1:35991, datanodeUuid=03545c32-c536-4081-8ede-49e42a2abaff, infoPort=45529, infoSecurePort=0, ipcPort=40211, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb16f01176ddfed33 with lease ID 0x4beb241b305ac8ed: from storage DS-3d8b146c-4d34-4c7d-937b-e6c1a2a385cf node DatanodeRegistration(127.0.0.1:35991, datanodeUuid=03545c32-c536-4081-8ede-49e42a2abaff, infoPort=45529, infoSecurePort=0, ipcPort=40211, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:24,942 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ffc270{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-35369-hadoop-hdfs-3_4_1-tests_jar-_-any-7176916226489138768/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:24,943 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2c1aeedb{HTTP/1.1, (http/1.1)}{localhost:35369} 2024-12-04T08:25:24,943 INFO [Time-limited test {}] server.Server(415): Started @116357ms 2024-12-04T08:25:24,945 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:24,996 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:24,999 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:25,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:25,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:25,000 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:25,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15751333{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:25,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65429201{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:25,116 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f41a928{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-41017-hadoop-hdfs-3_4_1-tests_jar-_-any-16489077583254173741/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:25,116 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c4c627f{HTTP/1.1, (http/1.1)}{localhost:41017} 2024-12-04T08:25:25,116 INFO [Time-limited test {}] server.Server(415): Started @116530ms 2024-12-04T08:25:25,118 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:25,181 WARN [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:25,181 WARN [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:25,204 WARN [Thread-845 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:25,206 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1268fb36c7153ea with lease ID 0x4beb241b305ac8ee: Processing first storage report for DS-bef57f47-ba7c-4401-a342-94b2822e5d95 from datanode DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:25,206 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1268fb36c7153ea with lease ID 0x4beb241b305ac8ee: from storage DS-bef57f47-ba7c-4401-a342-94b2822e5d95 node DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:25,206 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1268fb36c7153ea with lease ID 0x4beb241b305ac8ee: Processing first storage report for DS-a3dab5e1-eef7-4395-8763-a98ca6c0f11c from datanode DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:25,206 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1268fb36c7153ea with lease ID 0x4beb241b305ac8ee: from storage DS-a3dab5e1-eef7-4395-8763-a98ca6c0f11c node DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:25,229 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data9/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:25,229 WARN [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data10/current/BP-1202863867-172.17.0.2-1733300712426/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:25,246 WARN [Thread-868 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c9f8123d04eb54b with lease ID 0x4beb241b305ac8ef: Processing first storage report for DS-ad70d49b-2585-418d-8000-c262eef3401b from datanode DatanodeRegistration(127.0.0.1:42205, datanodeUuid=4252a863-7622-4535-a1b7-bfa71136dc80, infoPort=38287, infoSecurePort=0, ipcPort=36807, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c9f8123d04eb54b with lease ID 0x4beb241b305ac8ef: from storage DS-ad70d49b-2585-418d-8000-c262eef3401b node DatanodeRegistration(127.0.0.1:42205, datanodeUuid=4252a863-7622-4535-a1b7-bfa71136dc80, infoPort=38287, infoSecurePort=0, ipcPort=36807, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c9f8123d04eb54b with lease ID 0x4beb241b305ac8ef: Processing first storage report for DS-c5a56c0f-a473-4270-93c9-2a2f5291580b from datanode DatanodeRegistration(127.0.0.1:42205, datanodeUuid=4252a863-7622-4535-a1b7-bfa71136dc80, infoPort=38287, infoSecurePort=0, ipcPort=36807, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426) 2024-12-04T08:25:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c9f8123d04eb54b with lease ID 0x4beb241b305ac8ef: from storage DS-c5a56c0f-a473-4270-93c9-2a2f5291580b node DatanodeRegistration(127.0.0.1:42205, datanodeUuid=4252a863-7622-4535-a1b7-bfa71136dc80, infoPort=38287, infoSecurePort=0, ipcPort=36807, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:25,338 WARN [ResponseProcessor for block BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,338 WARN [ResponseProcessor for block BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,338 WARN [ResponseProcessor for block BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,338 WARN [ResponseProcessor for block BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,339 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta block BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:25,339 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:25,339 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:25,339 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:25,340 WARN [PacketResponder: BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35749] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,340 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-129105016_22 at /127.0.0.1:50190 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:35749:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50190 dst: /127.0.0.1:35749 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,340 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:50172 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35749:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50172 dst: /127.0.0.1:35749 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,340 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:45606 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45606 dst: /127.0.0.1:44007 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-702074429_22 at /127.0.0.1:45570 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45570 dst: /127.0.0.1:44007 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:45592 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45592 dst: /127.0.0.1:44007 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-129105016_22 at /127.0.0.1:45626 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45626 dst: /127.0.0.1:44007 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@85c422e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:25,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:50156 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35749:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50156 dst: /127.0.0.1:35749 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,342 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-702074429_22 at /127.0.0.1:50144 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35749:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50144 dst: /127.0.0.1:35749 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,342 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@293f42b5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:25,342 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:25,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@520d17ab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:25,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fe8f7b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:25,344 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:25,344 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1202863867-172.17.0.2-1733300712426 (Datanode Uuid 65dac4eb-f1a3-451a-9b5f-769df3baa54a) service to localhost/127.0.0.1:36333 2024-12-04T08:25:25,344 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:25,344 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:25,345 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data4/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:25,345 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data3/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:25,345 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:25,346 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@175cf5c8 {}] datanode.DataXceiver(331): 127.0.0.1:44007:DataXceiver error processing unknown operation src: /127.0.0.1:33058 dst: /127.0.0.1:44007 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,346 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,346 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,346 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,346 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-702074429_22 at /127.0.0.1:33054 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33054 dst: /127.0.0.1:44007 java.io.IOException: The stream is closed at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:25,347 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta block BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,353 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d790455{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:25,353 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@795c5052{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:25,354 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:25,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2df55a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:25,354 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@259c861e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:25,355 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:25,355 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:25,355 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1202863867-172.17.0.2-1733300712426 (Datanode Uuid dcc214a0-f297-48df-83a9-7fb8417285b3) service to localhost/127.0.0.1:36333 2024-12-04T08:25:25,355 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:25,356 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data1/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:25,356 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data2/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:25,356 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1., hostname=f5a5a857f5c5,37469,1733300713386, seqNum=2] 2024-12-04T08:25:25,362 ERROR [FSHLog-0-hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94-prefix:f5a5a857f5c5,37469,1733300713386 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,362 WARN [FSHLog-0-hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94-prefix:f5a5a857f5c5,37469,1733300713386 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,362 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,363 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C37469%2C1733300713386:(num 1733300713907) roll requested 2024-12-04T08:25:25,363 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 2024-12-04T08:25:25,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:25,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:25,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:25,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:25,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:25,369 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 2024-12-04T08:25:25,370 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,370 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:25,370 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45529:45529),(127.0.0.1/127.0.0.1:38287:38287)] 2024-12-04T08:25:25,371 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:25,371 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T08:25:25,372 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T08:25:25,372 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 2024-12-04T08:25:25,374 WARN [IPC Server handler 0 on default port 36333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009 2024-12-04T08:25:25,378 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 after 4ms 2024-12-04T08:25:25,468 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:26,508 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:27,371 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:27,372 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 2024-12-04T08:25:27,373 WARN [ResponseProcessor for block BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:27,373 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:27,373 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:46494 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:35991:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46494 dst: /127.0.0.1:35991 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:27,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:44398 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:42205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44398 dst: /127.0.0.1:42205 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:27,375 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@539abed5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:27,375 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2046b984{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:27,375 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:27,376 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fa27241{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:27,376 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@132d95f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:27,377 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:27,377 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1202863867-172.17.0.2-1733300712426 (Datanode Uuid 03545c32-c536-4081-8ede-49e42a2abaff) service to localhost/127.0.0.1:36333 2024-12-04T08:25:27,378 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data5/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:27,378 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data6/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:27,378 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:27,378 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:27,378 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:27,468 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:28,508 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:29,371 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:29,372 WARN [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]] 2024-12-04T08:25:29,372 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C37469%2C1733300713386:(num 1733300725363) roll requested 2024-12-04T08:25:29,373 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 2024-12-04T08:25:29,376 WARN [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:29,376 WARN [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:29,376 WARN [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741839_1021 2024-12-04T08:25:29,378 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 after 4006ms 2024-12-04T08:25:29,379 WARN [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:29,383 WARN [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35749 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:29,383 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:46600 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741840_1022 to mirror 127.0.0.1:35749 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:29,383 WARN [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:29,383 WARN [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741840_1022 2024-12-04T08:25:29,383 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:46600 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T08:25:29,383 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T08:25:29,383 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:46600 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46600 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:29,384 WARN [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:29,386 WARN [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35991 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:29,386 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:44424 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data10]'}, localName='127.0.0.1:42205', datanodeUuid='4252a863-7622-4535-a1b7-bfa71136dc80', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741841_1023 to mirror 127.0.0.1:35991 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:29,386 WARN [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:29,386 WARN [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741841_1023 2024-12-04T08:25:29,386 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:44424 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T08:25:29,386 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:44424 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:42205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44424 dst: /127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:29,386 WARN [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:29,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:29,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:29,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:29,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:29,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:29,391 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 2024-12-04T08:25:29,392 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45391:45391),(127.0.0.1/127.0.0.1:38287:38287)] 2024-12-04T08:25:29,392 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:29,392 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 is not closed yet, will try archiving it next time 2024-12-04T08:25:29,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42205 is added to blk_1073741838_1020 (size=3600) 2024-12-04T08:25:29,469 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:29,794 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:30,509 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,255 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@46df7b42[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42205, datanodeUuid=4252a863-7622-4535-a1b7-bfa71136dc80, infoPort=38287, infoSecurePort=0, ipcPort=36807, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741838_1020 to 127.0.0.1:35749 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:31,387 WARN [ResponseProcessor for block BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024 java.io.IOException: Bad response ERROR for BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024 from datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,387 WARN [DataStreamer for file /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 block BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:31,387 WARN [PacketResponder: BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42205] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:31,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:46612 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46612 dst: /127.0.0.1:37469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:31,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:44432 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:42205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44432 dst: /127.0.0.1:42205 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:31,389 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4f41a928{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:31,389 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c4c627f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:31,389 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:31,389 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65429201{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:31,390 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15751333{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:31,391 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:31,391 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:31,391 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1202863867-172.17.0.2-1733300712426 (Datanode Uuid 4252a863-7622-4535-a1b7-bfa71136dc80) service to localhost/127.0.0.1:36333 2024-12-04T08:25:31,391 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:31,391 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data9/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:31,391 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data10/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:31,392 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:31,392 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,392 WARN [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]] 2024-12-04T08:25:31,392 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C37469%2C1733300713386:(num 1733300729372) roll requested 2024-12-04T08:25:31,393 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.1733300731392 2024-12-04T08:25:31,395 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,395 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:31,395 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741843_1026 2024-12-04T08:25:31,396 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:31,397 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,397 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:31,397 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741844_1027 2024-12-04T08:25:31,398 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:31,399 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,399 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:31,399 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741845_1028 2024-12-04T08:25:31,400 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37469 {}] regionserver.HRegion(8855): Flush requested on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:31,401 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,401 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:31,401 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741846_1029 2024-12-04T08:25:31,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:25:31,402 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:31,402 WARN [IPC Server handler 2 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:31,402 WARN [IPC Server handler 2 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:31,403 WARN [IPC Server handler 2 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:31,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:31,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:31,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:31,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:31,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:31,406 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 with entries=6, filesize=6.11 KB; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300731392 2024-12-04T08:25:31,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741842_1025 (size=6261) 2024-12-04T08:25:31,415 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45391:45391)] 2024-12-04T08:25:31,415 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:31,415 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 is not closed yet, will try archiving it next time 2024-12-04T08:25:31,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/76848678d31d479280a4558fc9a83067 is 1080, key is row0002/info:/1733300727380/Put/seqid=0 2024-12-04T08:25:31,421 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,421 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:31,421 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741848_1031 2024-12-04T08:25:31,422 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:31,424 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55622 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741849_1032 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:31,425 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:31,425 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55622 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:31,425 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741849_1032 2024-12-04T08:25:31,425 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55622 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55622 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:31,425 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:31,427 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,427 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:31,427 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741850_1033 2024-12-04T08:25:31,427 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:31,429 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,429 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:31,429 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741851_1034 2024-12-04T08:25:31,429 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:31,430 WARN [IPC Server handler 2 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:31,430 WARN [IPC Server handler 2 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:31,430 WARN [IPC Server handler 2 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:31,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741852_1035 (size=10347) 2024-12-04T08:25:31,469 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:31,808 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:31,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/76848678d31d479280a4558fc9a83067 2024-12-04T08:25:31,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/76848678d31d479280a4558fc9a83067 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/76848678d31d479280a4558fc9a83067 2024-12-04T08:25:31,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/76848678d31d479280a4558fc9a83067, entries=5, sequenceid=11, filesize=10.1 K 2024-12-04T08:25:31,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 448ms, sequenceid=11, compaction requested=false 2024-12-04T08:25:31,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37469 {}] regionserver.HRegion(8855): Flush requested on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:32,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-12-04T08:25:32,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/43c42c4832d84426953f5d2f88cc5331 is 1080, key is row0007/info:/1733300731402/Put/seqid=0 2024-12-04T08:25:32,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55656 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741853_1036 to mirror 127.0.0.1:35749 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:32,040 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35749 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:32,040 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55656 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:32,040 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:32,040 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741853_1036 2024-12-04T08:25:32,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55656 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55656 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:32,041 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:32,042 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:32,042 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:32,042 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741854_1037 2024-12-04T08:25:32,043 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:32,044 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:32,044 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:32,044 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741855_1038 2024-12-04T08:25:32,045 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:32,045 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:32,046 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:32,046 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741856_1039 2024-12-04T08:25:32,046 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:32,047 WARN [IPC Server handler 1 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:32,047 WARN [IPC Server handler 1 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:32,047 WARN [IPC Server handler 1 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:32,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741857_1040 (size=12506) 2024-12-04T08:25:32,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/43c42c4832d84426953f5d2f88cc5331 2024-12-04T08:25:32,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/43c42c4832d84426953f5d2f88cc5331 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331 2024-12-04T08:25:32,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331, entries=7, sequenceid=24, filesize=12.2 K 2024-12-04T08:25:32,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 430ms, sequenceid=24, compaction requested=false 2024-12-04T08:25:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-12-04T08:25:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:32,463 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331 because midkey is the same as first or last row 2024-12-04T08:25:32,509 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,415 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,416 WARN [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]] 2024-12-04T08:25:33,416 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C37469%2C1733300713386:(num 1733300731392) roll requested 2024-12-04T08:25:33,416 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.1733300733416 2024-12-04T08:25:33,419 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,419 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:33,419 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741858_1041 2024-12-04T08:25:33,420 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:33,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55662 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741859_1042 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,422 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,422 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55662 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T08:25:33,422 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:33,422 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741859_1042 2024-12-04T08:25:33,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55662 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55662 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,423 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:33,425 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44007 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,425 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55678 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741860_1043 to mirror 127.0.0.1:44007 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,425 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:33,425 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55678 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T08:25:33,425 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741860_1043 2024-12-04T08:25:33,425 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55678 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55678 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,425 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:33,427 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55694 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741861_1044 to mirror 127.0.0.1:35749 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,427 WARN [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35749 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,427 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55694 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T08:25:33,427 WARN [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:33,427 WARN [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741861_1044 2024-12-04T08:25:33,427 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55694 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55694 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,428 WARN [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:33,429 WARN [IPC Server handler 3 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:33,429 WARN [IPC Server handler 3 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:33,429 WARN [IPC Server handler 3 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:33,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:33,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:33,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:33,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:33,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:33,432 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300731392 with entries=18, filesize=18.21 KB; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300733416 2024-12-04T08:25:33,433 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45391:45391)] 2024-12-04T08:25:33,433 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:33,433 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300731392 is not closed yet, will try archiving it next time 2024-12-04T08:25:33,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741847_1030 (size=18655) 2024-12-04T08:25:33,434 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs/f5a5a857f5c5%2C37469%2C1733300713386.1733300725363 2024-12-04T08:25:33,434 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 is not closed yet, will try archiving it next time 2024-12-04T08:25:33,435 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs/f5a5a857f5c5%2C37469%2C1733300713386.1733300729372 2024-12-04T08:25:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37469 {}] regionserver.HRegion(8855): Flush requested on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:33,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T08:25:33,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/5b1196d0834f4cc4933f956d8a6aaf3b is 1079, key is tmprow/info:/1733300733450/Put/seqid=0 2024-12-04T08:25:33,458 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55714 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741863_1046 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,458 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,458 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55714 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:33,458 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:33,458 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741863_1046 2024-12-04T08:25:33,458 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55714 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55714 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,459 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:33,460 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,460 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:33,460 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741864_1047 2024-12-04T08:25:33,461 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:33,462 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,462 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:33,462 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741865_1048 2024-12-04T08:25:33,462 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:33,465 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35991 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55724 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741866_1049 to mirror 127.0.0.1:35991 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,465 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:33,465 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55724 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:33,465 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741866_1049 2024-12-04T08:25:33,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55724 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55724 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,465 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:33,466 WARN [IPC Server handler 4 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:33,466 WARN [IPC Server handler 4 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:33,466 WARN [IPC Server handler 4 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:33,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741867_1050 (size=6027) 2024-12-04T08:25:33,469 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/5b1196d0834f4cc4933f956d8a6aaf3b 2024-12-04T08:25:33,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/5b1196d0834f4cc4933f956d8a6aaf3b as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/5b1196d0834f4cc4933f956d8a6aaf3b 2024-12-04T08:25:33,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/5b1196d0834f4cc4933f956d8a6aaf3b, entries=1, sequenceid=34, filesize=5.9 K 2024-12-04T08:25:33,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 432ms, sequenceid=34, compaction requested=true 2024-12-04T08:25:33,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-12-04T08:25:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331 because midkey is the same as first or last row 2024-12-04T08:25:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3af4b7ac06a411c8ccf86f339fcc9ac1:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:25:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:25:33,884 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:25:33,885 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:25:33,885 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HStore(1541): 3af4b7ac06a411c8ccf86f339fcc9ac1/info is initiating minor compaction (all files) 2024-12-04T08:25:33,885 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3af4b7ac06a411c8ccf86f339fcc9ac1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:33,886 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/76848678d31d479280a4558fc9a83067, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/5b1196d0834f4cc4933f956d8a6aaf3b] into tmpdir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp, totalSize=28.2 K 2024-12-04T08:25:33,886 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76848678d31d479280a4558fc9a83067, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733300727380 2024-12-04T08:25:33,886 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c42c4832d84426953f5d2f88cc5331, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733300731402 2024-12-04T08:25:33,887 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b1196d0834f4cc4933f956d8a6aaf3b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733300733450 2024-12-04T08:25:33,899 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3af4b7ac06a411c8ccf86f339fcc9ac1#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:25:33,900 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/97e4b9c936e54576acc92dbf6228314c is 1080, key is row0002/info:/1733300727380/Put/seqid=0 2024-12-04T08:25:33,902 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,902 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:33,902 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741868_1051 2024-12-04T08:25:33,902 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:33,904 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,904 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:33,904 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741869_1052 2024-12-04T08:25:33,904 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:33,906 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,906 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55758 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741870_1053 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,907 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:33,907 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741870_1053 2024-12-04T08:25:33,907 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55758 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:33,907 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55758 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55758 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,907 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:33,910 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35749 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:33,910 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55764 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741871_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741871_1054 to mirror 127.0.0.1:35749 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,911 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:33,911 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741871_1054 2024-12-04T08:25:33,911 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55764 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741871_1054] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:33,911 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55764 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741871_1054] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55764 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:33,911 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:33,912 WARN [IPC Server handler 3 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:33,912 WARN [IPC Server handler 3 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:33,912 WARN [IPC Server handler 3 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:33,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741872_1055 (size=17994) 2024-12-04T08:25:34,207 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@2b42e84b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741842_1025 to 127.0.0.1:42205 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:34,207 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741852_1035 to 127.0.0.1:42205 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:34,322 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/97e4b9c936e54576acc92dbf6228314c as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c 2024-12-04T08:25:34,328 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3af4b7ac06a411c8ccf86f339fcc9ac1/info of 3af4b7ac06a411c8ccf86f339fcc9ac1 into 97e4b9c936e54576acc92dbf6228314c(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:25:34,328 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:34,328 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1., storeName=3af4b7ac06a411c8ccf86f339fcc9ac1/info, priority=13, startTime=1733300733884; duration=0sec 2024-12-04T08:25:34,328 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-04T08:25:34,328 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:34,328 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c because midkey is the same as first or last row 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c because midkey is the same as first or last row 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c because midkey is the same as first or last row 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:25:34,329 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3af4b7ac06a411c8ccf86f339fcc9ac1:info 2024-12-04T08:25:34,509 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37469 {}] regionserver.HRegion(8855): Flush requested on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:34,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T08:25:34,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/e91d01f04e364518b23db034804038da is 1079, key is tmprow/info:/1733300734869/Put/seqid=0 2024-12-04T08:25:34,878 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35749 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:34,878 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55786 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741873_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741873_1056 to mirror 127.0.0.1:35749 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:34,878 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]) is bad. 2024-12-04T08:25:34,878 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55786 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741873_1056] {}] datanode.BlockReceiver(316): Block 1073741873 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:34,878 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741873_1056 2024-12-04T08:25:34,878 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55786 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741873_1056] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55786 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:34,879 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35749,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK] 2024-12-04T08:25:34,880 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:34,880 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:34,880 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741874_1057 2024-12-04T08:25:34,881 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:34,882 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:34,882 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:34,882 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741875_1058 2024-12-04T08:25:34,882 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:34,884 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:34,884 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55798 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741876_1059 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:34,884 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:34,884 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741876_1059 2024-12-04T08:25:34,884 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55798 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:34,885 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:55798 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55798 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:34,885 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:34,885 WARN [IPC Server handler 2 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-04T08:25:34,886 WARN [IPC Server handler 2 on default port 36333 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-04T08:25:34,886 WARN [IPC Server handler 2 on default port 36333 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-04T08:25:34,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741877_1060 (size=6027) 2024-12-04T08:25:35,207 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741857_1040 to 127.0.0.1:42205 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:35,207 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@2b42e84b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741847_1030 to 127.0.0.1:35749 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:35,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/e91d01f04e364518b23db034804038da 2024-12-04T08:25:35,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/e91d01f04e364518b23db034804038da as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/e91d01f04e364518b23db034804038da 2024-12-04T08:25:35,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/e91d01f04e364518b23db034804038da, entries=1, sequenceid=45, filesize=5.9 K 2024-12-04T08:25:35,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 432ms, sequenceid=45, compaction requested=false 2024-12-04T08:25:35,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:35,302 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-12-04T08:25:35,302 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:35,302 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c because midkey is the same as first or last row 2024-12-04T08:25:35,434 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:35,434 WARN [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-12-04T08:25:35,469 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:35,485 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:35,488 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:35,488 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:35,488 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:35,488 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:35,489 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c92a539{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:35,489 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@743041f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:35,603 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@719d00c0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/java.io.tmpdir/jetty-localhost-40553-hadoop-hdfs-3_4_1-tests_jar-_-any-3085080610868179318/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:35,604 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@792fa80c{HTTP/1.1, (http/1.1)}{localhost:40553} 2024-12-04T08:25:35,604 INFO [Time-limited test {}] server.Server(415): Started @127017ms 2024-12-04T08:25:35,605 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:35,705 WARN [Thread-986 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:35,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65d86531f179ac57 with lease ID 0x4beb241b305ac8f0: from storage DS-fb1b7e16-a807-4b59-88c3-8909635b5810 node DatanodeRegistration(127.0.0.1:37871, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=37123, infoSecurePort=0, ipcPort=40839, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T08:25:35,713 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65d86531f179ac57 with lease ID 0x4beb241b305ac8f0: from storage DS-35456c5a-541d-4df1-8d1a-a1e0d1aa41f4 node DatanodeRegistration(127.0.0.1:37871, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=37123, infoSecurePort=0, ipcPort=40839, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:36,510 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:37,208 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@5b4614c1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741872_1055 to 127.0.0.1:35991 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:37,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741867_1050 (size=6027) 2024-12-04T08:25:37,434 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:37,470 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:38,209 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@2b42e84b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741877_1060 to 127.0.0.1:35991 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:38,510 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:39,434 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:39,470 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:40,510 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:41,435 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:41,470 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:42,511 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,301 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:25:43,435 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,471 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,654 ERROR [FSHLog-0-hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData-prefix:f5a5a857f5c5,41417,1733300713326 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,654 WARN [FSHLog-0-hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData-prefix:f5a5a857f5c5,41417,1733300713326 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,654 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C41417%2C1733300713326:(num 1733300713493) roll requested 2024-12-04T08:25:43,655 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C41417%2C1733300713326.1733300743654 2024-12-04T08:25:43,659 WARN [Thread-1007 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,659 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-702074429_22 at /127.0.0.1:56242 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741878_1061] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data4]'}, localName='127.0.0.1:37871', datanodeUuid='65dac4eb-f1a3-451a-9b5f-769df3baa54a', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741878_1061 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:43,659 WARN [Thread-1007 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:43,659 WARN [Thread-1007 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741878_1061 2024-12-04T08:25:43,659 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-702074429_22 at /127.0.0.1:56242 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741878_1061] {}] datanode.BlockReceiver(316): Block 1073741878 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-04T08:25:43,659 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-702074429_22 at /127.0.0.1:56242 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741878_1061] {}] datanode.DataXceiver(331): 127.0.0.1:37871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56242 dst: /127.0.0.1:37871 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:43,660 WARN [Thread-1007 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:43,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:43,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:43,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:43,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:43,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:43,664 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300743654 2024-12-04T08:25:43,665 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,665 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:43,665 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 2024-12-04T08:25:43,665 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37123:37123),(127.0.0.1/127.0.0.1:45391:45391)] 2024-12-04T08:25:43,665 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 is not closed yet, will try archiving it next time 2024-12-04T08:25:43,665 WARN [IPC Server handler 2 on default port 36333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 has not been closed. Lease recovery is in progress. RecoveryId = 1063 for block blk_1073741830_1006 2024-12-04T08:25:43,666 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 after 1ms 2024-12-04T08:25:44,511 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:45,435 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:45,728 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@e3bfd34 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:44007,null,null]) java.net.ConnectException: Call From f5a5a857f5c5/172.17.0.2 to localhost:45357 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T08:25:45,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741833_1019 (size=455) 2024-12-04T08:25:46,393 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs/f5a5a857f5c5%2C37469%2C1733300713386.1733300713907 2024-12-04T08:25:46,395 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300731392 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs/f5a5a857f5c5%2C37469%2C1733300713386.1733300731392 2024-12-04T08:25:46,511 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:46,708 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3f7fb7ea[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37871, datanodeUuid=65dac4eb-f1a3-451a-9b5f-769df3baa54a, infoPort=37123, infoSecurePort=0, ipcPort=40839, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741833_1019 to 127.0.0.1:35991 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:47,436 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:47,667 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/WALs/f5a5a857f5c5,41417,1733300713326/f5a5a857f5c5%2C41417%2C1733300713326.1733300713493 after 4002ms 2024-12-04T08:25:48,512 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:49,436 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:50,512 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,338 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.1733300751338 2024-12-04T08:25:51,341 WARN [Thread-1019 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1064 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,341 WARN [Thread-1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741880_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:51,341 WARN [Thread-1019 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741880_1064 2024-12-04T08:25:51,342 WARN [Thread-1019 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:51,343 WARN [Thread-1019 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1065 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,343 WARN [Thread-1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741881_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:51,343 WARN [Thread-1019 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741881_1065 2024-12-04T08:25:51,343 WARN [Thread-1019 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:51,344 WARN [Thread-1019 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1066 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,345 WARN [Thread-1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741882_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:51,345 WARN [Thread-1019 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741882_1066 2024-12-04T08:25:51,345 WARN [Thread-1019 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:51,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,350 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300733416 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300751338 2024-12-04T08:25:51,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37123:37123),(127.0.0.1/127.0.0.1:45391:45391)] 2024-12-04T08:25:51,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300733416 is not closed yet, will try archiving it next time 2024-12-04T08:25:51,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741862_1045 (size=13591) 2024-12-04T08:25:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37469 {}] regionserver.HRegion(8855): Flush requested on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:51,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-04T08:25:51,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/b180c831155b4b259ce42c7fe71305b6 is 1080, key is row0013/info:/1733300751352/Put/seqid=0 2024-12-04T08:25:51,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741884_1068 (size=11421) 2024-12-04T08:25:51,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741884_1068 (size=11421) 2024-12-04T08:25:51,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/b180c831155b4b259ce42c7fe71305b6 2024-12-04T08:25:51,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/b180c831155b4b259ce42c7fe71305b6 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/b180c831155b4b259ce42c7fe71305b6 2024-12-04T08:25:51,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/b180c831155b4b259ce42c7fe71305b6, entries=6, sequenceid=55, filesize=11.2 K 2024-12-04T08:25:51,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=6.30 KB/6455 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 24ms, sequenceid=55, compaction requested=true 2024-12-04T08:25:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.6 K, sizeToCheck=16.0 K 2024-12-04T08:25:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c because midkey is the same as first or last row 2024-12-04T08:25:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3af4b7ac06a411c8ccf86f339fcc9ac1:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:25:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:25:51,386 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:25:51,387 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:25:51,387 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HStore(1541): 3af4b7ac06a411c8ccf86f339fcc9ac1/info is initiating minor compaction (all files) 2024-12-04T08:25:51,387 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3af4b7ac06a411c8ccf86f339fcc9ac1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:51,387 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/e91d01f04e364518b23db034804038da, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/b180c831155b4b259ce42c7fe71305b6] into tmpdir=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp, totalSize=34.6 K 2024-12-04T08:25:51,388 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e4b9c936e54576acc92dbf6228314c, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733300727380 2024-12-04T08:25:51,388 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91d01f04e364518b23db034804038da, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733300734869 2024-12-04T08:25:51,388 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] compactions.Compactor(225): Compacting b180c831155b4b259ce42c7fe71305b6, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733300735275 2024-12-04T08:25:51,403 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3af4b7ac06a411c8ccf86f339fcc9ac1#info#compaction#24 average throughput is 8.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:25:51,404 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/4b723975cbd942ce93eef896a36c3594 is 1080, key is row0002/info:/1733300727380/Put/seqid=0 2024-12-04T08:25:51,405 WARN [Thread-1035 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1069 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,406 WARN [Thread-1035 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741885_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:51,406 WARN [Thread-1035 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741885_1069 2024-12-04T08:25:51,406 WARN [Thread-1035 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:51,408 WARN [Thread-1035 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1070 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35991 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,408 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:43870 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741886_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741886_1070 to mirror 127.0.0.1:35991 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,408 WARN [Thread-1035 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741886_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:51,408 WARN [Thread-1035 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741886_1070 2024-12-04T08:25:51,409 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:43870 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741886_1070] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:51,409 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:43870 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741886_1070] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43870 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,409 WARN [Thread-1035 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:51,411 WARN [Thread-1035 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1071 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42205 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,411 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:43872 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741887_1071] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8]'}, localName='127.0.0.1:37469', datanodeUuid='72a118e3-ab4e-4bd9-87d5-53dd9586738e', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741887_1071 to mirror 127.0.0.1:42205 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,411 WARN [Thread-1035 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741887_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:51,411 WARN [Thread-1035 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741887_1071 2024-12-04T08:25:51,411 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:43872 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741887_1071] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:51,411 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:43872 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741887_1071] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43872 dst: /127.0.0.1:37469 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,412 WARN [Thread-1035 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:51,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741888_1072 (size=23502) 2024-12-04T08:25:51,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741888_1072 (size=23502) 2024-12-04T08:25:51,423 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/4b723975cbd942ce93eef896a36c3594 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/4b723975cbd942ce93eef896a36c3594 2024-12-04T08:25:51,430 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3af4b7ac06a411c8ccf86f339fcc9ac1/info of 3af4b7ac06a411c8ccf86f339fcc9ac1 into 4b723975cbd942ce93eef896a36c3594(size=23.0 K), total size for store is 23.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:25:51,430 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: 2024-12-04T08:25:51,430 INFO [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1., storeName=3af4b7ac06a411c8ccf86f339fcc9ac1/info, priority=13, startTime=1733300751386; duration=0sec 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/4b723975cbd942ce93eef896a36c3594 because midkey is the same as first or last row 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/4b723975cbd942ce93eef896a36c3594 because midkey is the same as first or last row 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/4b723975cbd942ce93eef896a36c3594 because midkey is the same as first or last row 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:25:51,431 DEBUG [RS:0;f5a5a857f5c5:37469-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3af4b7ac06a411c8ccf86f339fcc9ac1:info 2024-12-04T08:25:51,436 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,436 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-12-04T08:25:51,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:25:51,574 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:25:51,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:51,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:51,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:51,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:25:51,575 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:25:51,575 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1403695941, stopped=false 2024-12-04T08:25:51,575 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,41417,1733300713326 2024-12-04T08:25:51,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:51,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:51,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:51,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:51,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:51,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:51,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:25:51,578 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:25:51,578 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:51,578 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:51,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:51,578 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:51,578 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:51,578 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,37469,1733300713386' ***** 2024-12-04T08:25:51,578 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:25:51,578 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,35147,1733300714444' ***** 2024-12-04T08:25:51,578 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:25:51,579 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:25:51,579 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:25:51,579 INFO [RS:1;f5a5a857f5c5:35147 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:25:51,579 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:25:51,579 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T08:25:51,579 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:25:51,579 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T08:25:51,579 INFO [RS:1;f5a5a857f5c5:35147 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:25:51,579 INFO [RS:0;f5a5a857f5c5:37469 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:25:51,579 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:51,579 INFO [RS:0;f5a5a857f5c5:37469 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:25:51,579 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:51,579 INFO [RS:1;f5a5a857f5c5:35147 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f5a5a857f5c5:35147. 2024-12-04T08:25:51,579 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(3091): Received CLOSE for 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:51,579 DEBUG [RS:1;f5a5a857f5c5:35147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:51,579 DEBUG [RS:1;f5a5a857f5c5:35147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:51,580 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,35147,1733300714444; all regions closed. 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:37469. 2024-12-04T08:25:51,580 DEBUG [RS:0;f5a5a857f5c5:37469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:25:51,580 DEBUG [RS:0;f5a5a857f5c5:37469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:25:51,580 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3af4b7ac06a411c8ccf86f339fcc9ac1, disabling compactions & flushes 2024-12-04T08:25:51,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:25:51,580 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:51,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,580 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:51,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,580 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. after waiting 0 ms 2024-12-04T08:25:51,580 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T08:25:51,580 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:51,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,580 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 3af4b7ac06a411c8ccf86f339fcc9ac1=TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.} 2024-12-04T08:25:51,580 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:51,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,581 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:25:51,581 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1 1/1 column families, dataSize=6.30 KB heapSize=7 KB 2024-12-04T08:25:51,581 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:25:51,581 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:25:51,581 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:25:51,581 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:25:51,581 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-12-04T08:25:51,581 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,581 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,581 ERROR [FSHLog-0-hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94-prefix:f5a5a857f5c5,37469,1733300713386.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,581 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 2024-12-04T08:25:51,581 WARN [FSHLog-0-hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94-prefix:f5a5a857f5c5,37469,1733300713386.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,581 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C37469%2C1733300713386.meta:.meta(num 1733300714278) roll requested 2024-12-04T08:25:51,582 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300751582.meta 2024-12-04T08:25:51,582 WARN [IPC Server handler 1 on default port 36333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741837_1013 2024-12-04T08:25:51,582 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 after 1ms 2024-12-04T08:25:51,584 WARN [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1074 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,584 WARN [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741889_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:51,585 WARN [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741889_1074 2024-12-04T08:25:51,585 WARN [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:51,586 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/3e0209a08c8b4c2099ec485a090e050c is 1080, key is row0018/info:/1733300751362/Put/seqid=0 2024-12-04T08:25:51,586 WARN [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1075 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,586 WARN [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741890_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:51,586 WARN [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741890_1075 2024-12-04T08:25:51,587 WARN [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:51,587 WARN [Thread-1044 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1076 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,588 WARN [Thread-1044 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741891_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:51,588 WARN [Thread-1044 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741891_1076 2024-12-04T08:25:51,588 WARN [Thread-1044 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:51,590 WARN [Thread-1044 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1078 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35991 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,590 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:37134 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741893_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data4]'}, localName='127.0.0.1:37871', datanodeUuid='65dac4eb-f1a3-451a-9b5f-769df3baa54a', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741893_1078 to mirror 127.0.0.1:35991 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,590 WARN [Thread-1044 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741893_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:51,590 WARN [Thread-1044 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741893_1078 2024-12-04T08:25:51,590 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:37134 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741893_1078] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:51,590 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:37134 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741893_1078] {}] datanode.DataXceiver(331): 127.0.0.1:37871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37134 dst: /127.0.0.1:37871 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,591 WARN [Thread-1044 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:51,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:51,592 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300751582.meta 2024-12-04T08:25:51,592 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,592 WARN [Thread-1044 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1079 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,592 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,592 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta 2024-12-04T08:25:51,592 WARN [Thread-1044 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741894_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:51,592 WARN [Thread-1044 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741894_1079 2024-12-04T08:25:51,592 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45391:45391),(127.0.0.1/127.0.0.1:37123:37123)] 2024-12-04T08:25:51,592 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta is not closed yet, will try archiving it next time 2024-12-04T08:25:51,592 WARN [IPC Server handler 0 on default port 36333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741834_1010 2024-12-04T08:25:51,593 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta after 1ms 2024-12-04T08:25:51,593 WARN [Thread-1044 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:51,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741895_1081 (size=11421) 2024-12-04T08:25:51,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741895_1081 (size=11421) 2024-12-04T08:25:51,616 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/info/edf75e57426248b0acd623476a6353c0 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1./info:regioninfo/1733300714913/Put/seqid=0 2024-12-04T08:25:51,618 WARN [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1082 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,618 WARN [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741896_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:51,618 WARN [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741896_1082 2024-12-04T08:25:51,619 WARN [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:51,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741897_1083 (size=7089) 2024-12-04T08:25:51,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741897_1083 (size=7089) 2024-12-04T08:25:51,624 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/info/edf75e57426248b0acd623476a6353c0 2024-12-04T08:25:51,652 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/ns/b17623491142454797b348bc779a58a3 is 43, key is default/ns:d/1733300714338/Put/seqid=0 2024-12-04T08:25:51,654 WARN [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1084 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,655 WARN [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741898_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK], DatanodeInfoWithStorage[127.0.0.1:37469,DS-bef57f47-ba7c-4401-a342-94b2822e5d95,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]) is bad. 2024-12-04T08:25:51,655 WARN [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741898_1084 2024-12-04T08:25:51,655 WARN [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK] 2024-12-04T08:25:51,657 WARN [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1085 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35991 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,657 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:37168 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741899_1085] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data4]'}, localName='127.0.0.1:37871', datanodeUuid='65dac4eb-f1a3-451a-9b5f-769df3baa54a', xmitsInProgress=0}:Exception transferring block BP-1202863867-172.17.0.2-1733300712426:blk_1073741899_1085 to mirror 127.0.0.1:35991 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,657 WARN [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741899_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37871,DS-fb1b7e16-a807-4b59-88c3-8909635b5810,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:51,657 WARN [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741899_1085 2024-12-04T08:25:51,657 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:37168 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741899_1085] {}] datanode.BlockReceiver(316): Block 1073741899 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-04T08:25:51,658 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1432879736_22 at /127.0.0.1:37168 [Receiving block BP-1202863867-172.17.0.2-1733300712426:blk_1073741899_1085] {}] datanode.DataXceiver(331): 127.0.0.1:37871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37168 dst: /127.0.0.1:37871 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:51,658 WARN [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:51,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741900_1086 (size=5153) 2024-12-04T08:25:51,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741900_1086 (size=5153) 2024-12-04T08:25:51,662 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/ns/b17623491142454797b348bc779a58a3 2024-12-04T08:25:51,683 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/table/2a29ca6aeef74937bbd41c0625031209 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733300714923/Put/seqid=0 2024-12-04T08:25:51,684 WARN [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1087 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,685 WARN [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741901_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK], DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK]) is bad. 2024-12-04T08:25:51,685 WARN [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741901_1087 2024-12-04T08:25:51,685 WARN [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44007,DS-ca06bcae-5faf-4549-b4ad-e902365b392d,DISK] 2024-12-04T08:25:51,686 WARN [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1088 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:25:51,687 WARN [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1202863867-172.17.0.2-1733300712426:blk_1073741902_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK], DatanodeInfoWithStorage[127.0.0.1:42205,DS-ad70d49b-2585-418d-8000-c262eef3401b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK]) is bad. 2024-12-04T08:25:51,687 WARN [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1202863867-172.17.0.2-1733300712426:blk_1073741902_1088 2024-12-04T08:25:51,687 WARN [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35991,DS-6f9c8215-bade-4a6a-a2b1-83b86eb3f74d,DISK] 2024-12-04T08:25:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741903_1089 (size=5424) 2024-12-04T08:25:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741903_1089 (size=5424) 2024-12-04T08:25:51,692 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/table/2a29ca6aeef74937bbd41c0625031209 2024-12-04T08:25:51,698 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/info/edf75e57426248b0acd623476a6353c0 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/info/edf75e57426248b0acd623476a6353c0 2024-12-04T08:25:51,703 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/info/edf75e57426248b0acd623476a6353c0, entries=10, sequenceid=11, filesize=6.9 K 2024-12-04T08:25:51,704 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/ns/b17623491142454797b348bc779a58a3 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/ns/b17623491142454797b348bc779a58a3 2024-12-04T08:25:51,709 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/ns/b17623491142454797b348bc779a58a3, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T08:25:51,710 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/.tmp/table/2a29ca6aeef74937bbd41c0625031209 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/table/2a29ca6aeef74937bbd41c0625031209 2024-12-04T08:25:51,715 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/table/2a29ca6aeef74937bbd41c0625031209, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T08:25:51,717 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 135ms, sequenceid=11, compaction requested=false 2024-12-04T08:25:51,721 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T08:25:51,722 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:25:51,722 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:51,722 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300751580Running coprocessor pre-close hooks at 1733300751580Disabling compacts and flushes for region at 1733300751580Disabling writes for close at 1733300751581 (+1 ms)Obtaining lock to block concurrent updates at 1733300751581Preparing flush snapshotting stores in 1588230740 at 1733300751581Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733300751581Flushing stores of hbase:meta,,1.1588230740 at 1733300751593 (+12 ms)Flushing 1588230740/info: creating writer at 1733300751593Flushing 1588230740/info: appending metadata at 1733300751616 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733300751616Flushing 1588230740/ns: creating writer at 1733300751631 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733300751652 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733300751652Flushing 1588230740/table: creating writer at 1733300751668 (+16 ms)Flushing 1588230740/table: appending metadata at 1733300751682 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733300751682Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24f7eb09: reopening flushed file at 1733300751697 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b1c11d4: reopening flushed file at 1733300751703 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@309dc115: reopening flushed file at 1733300751709 (+6 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 135ms, sequenceid=11, compaction requested=false at 1733300751717 (+8 ms)Writing region close event to WAL at 1733300751718 (+1 ms)Running coprocessor post-close hooks at 1733300751722 (+4 ms)Closed at 1733300751722 2024-12-04T08:25:51,722 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:51,752 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.1733300733416 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs/f5a5a857f5c5%2C37469%2C1733300713386.1733300733416 2024-12-04T08:25:51,768 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:51,781 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1351): Waiting on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:51,839 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T08:25:51,839 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T08:25:51,981 DEBUG [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1351): Waiting on 3af4b7ac06a411c8ccf86f339fcc9ac1 2024-12-04T08:25:51,999 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/3e0209a08c8b4c2099ec485a090e050c 2024-12-04T08:25:52,005 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/.tmp/info/3e0209a08c8b4c2099ec485a090e050c as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/3e0209a08c8b4c2099ec485a090e050c 2024-12-04T08:25:52,011 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/3e0209a08c8b4c2099ec485a090e050c, entries=6, sequenceid=65, filesize=11.2 K 2024-12-04T08:25:52,012 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6455, heapSize ~6.98 KB/7152, currentSize=0 B/0 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 432ms, sequenceid=65, compaction requested=false 2024-12-04T08:25:52,013 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/76848678d31d479280a4558fc9a83067, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/5b1196d0834f4cc4933f956d8a6aaf3b, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/e91d01f04e364518b23db034804038da, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/b180c831155b4b259ce42c7fe71305b6] to archive 2024-12-04T08:25:52,014 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T08:25:52,016 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/76848678d31d479280a4558fc9a83067 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/76848678d31d479280a4558fc9a83067 2024-12-04T08:25:52,017 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/43c42c4832d84426953f5d2f88cc5331 2024-12-04T08:25:52,018 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/97e4b9c936e54576acc92dbf6228314c 2024-12-04T08:25:52,019 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/5b1196d0834f4cc4933f956d8a6aaf3b to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/5b1196d0834f4cc4933f956d8a6aaf3b 2024-12-04T08:25:52,021 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/e91d01f04e364518b23db034804038da to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/e91d01f04e364518b23db034804038da 2024-12-04T08:25:52,022 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/b180c831155b4b259ce42c7fe71305b6 to hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/info/b180c831155b4b259ce42c7fe71305b6 2024-12-04T08:25:52,022 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f5a5a857f5c5:41417 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T08:25:52,023 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [76848678d31d479280a4558fc9a83067=10347, 43c42c4832d84426953f5d2f88cc5331=12506, 97e4b9c936e54576acc92dbf6228314c=17994, 5b1196d0834f4cc4933f956d8a6aaf3b=6027, e91d01f04e364518b23db034804038da=6027, b180c831155b4b259ce42c7fe71305b6=11421] 2024-12-04T08:25:52,027 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3af4b7ac06a411c8ccf86f339fcc9ac1/recovered.edits/68.seqid, newMaxSeqId=68, maxSeqId=1 2024-12-04T08:25:52,027 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:52,028 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3af4b7ac06a411c8ccf86f339fcc9ac1: Waiting for close lock at 1733300751580Running coprocessor pre-close hooks at 1733300751580Disabling compacts and flushes for region at 1733300751580Disabling writes for close at 1733300751580Obtaining lock to block concurrent updates at 1733300751581 (+1 ms)Preparing flush snapshotting stores in 3af4b7ac06a411c8ccf86f339fcc9ac1 at 1733300751581Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1., syncing WAL and waiting on mvcc, flushsize=dataSize=6455, getHeapSize=7152, getOffHeapSize=0, getCellsCount=6 at 1733300751581Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. at 1733300751582 (+1 ms)Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1/info: creating writer at 1733300751582Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1/info: appending metadata at 1733300751585 (+3 ms)Flushing 3af4b7ac06a411c8ccf86f339fcc9ac1/info: closing flushed file at 1733300751585Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ec6295: reopening flushed file at 1733300752005 (+420 ms)Finished flush of dataSize ~6.30 KB/6455, heapSize ~6.98 KB/7152, currentSize=0 B/0 for 3af4b7ac06a411c8ccf86f339fcc9ac1 in 432ms, sequenceid=65, compaction requested=false at 1733300752012 (+7 ms)Writing region close event to WAL at 1733300752023 (+11 ms)Running coprocessor post-close hooks at 1733300752027 (+4 ms)Closed at 1733300752027 2024-12-04T08:25:52,028 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733300714541.3af4b7ac06a411c8ccf86f339fcc9ac1. 2024-12-04T08:25:52,181 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,37469,1733300713386; all regions closed. 2024-12-04T08:25:52,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:52,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:52,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:52,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:52,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:52,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741892_1077 (size=825) 2024-12-04T08:25:52,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741892_1077 (size=825) 2024-12-04T08:25:52,208 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@2b42e84b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37469, datanodeUuid=72a118e3-ab4e-4bd9-87d5-53dd9586738e, infoPort=45391, infoSecurePort=0, ipcPort=41183, storageInfo=lv=-57;cid=testClusterID;nsid=1645468702;c=1733300712426):Failed to transfer BP-1202863867-172.17.0.2-1733300712426:blk_1073741862_1045 to 127.0.0.1:42205 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:52,509 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:53,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-04T08:25:53,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:25:53,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T08:25:54,357 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T08:25:54,357 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T08:25:55,583 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 after 4002ms 2024-12-04T08:25:55,594 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta after 4001ms 2024-12-04T08:25:55,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:55,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741835_1011 (size=393) 2024-12-04T08:25:55,732 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47b99e9d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1202863867-172.17.0.2-1733300712426:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:44007,null,null]) java.net.ConnectException: Call From f5a5a857f5c5/172.17.0.2 to localhost:45357 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T08:25:56,581 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-04T08:25:56,583 DEBUG [RS:1;f5a5a857f5c5:35147 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs 2024-12-04T08:25:56,583 INFO [RS:1;f5a5a857f5c5:35147 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C35147%2C1733300714444:(num 1733300714642) 2024-12-04T08:25:56,583 DEBUG [RS:1;f5a5a857f5c5:35147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:56,583 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:56,583 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:56,584 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:56,584 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:25:56,584 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:56,584 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:25:56,584 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:25:56,584 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:56,584 INFO [RS:1;f5a5a857f5c5:35147 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35147 2024-12-04T08:25:56,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,35147,1733300714444 2024-12-04T08:25:56,586 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:56,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:56,588 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,35147,1733300714444] 2024-12-04T08:25:56,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:56,590 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,35147,1733300714444 already deleted, retry=false 2024-12-04T08:25:56,590 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,35147,1733300714444 expired; onlineServers=1 2024-12-04T08:25:56,688 INFO [RS:1;f5a5a857f5c5:35147 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:56,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:56,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x1017c93e1e80002, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:56,688 INFO [RS:1;f5a5a857f5c5:35147 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,35147,1733300714444; zookeeper connection closed. 2024-12-04T08:25:56,689 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c44c8c7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c44c8c7 2024-12-04T08:25:56,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:56,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:56,723 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,733 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,734 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,734 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,734 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,734 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,739 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:56,741 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,183 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-04T08:25:57,186 DEBUG [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs 2024-12-04T08:25:57,186 INFO [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C37469%2C1733300713386.meta:.meta(num 1733300751582) 2024-12-04T08:25:57,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741883_1067 (size=15140) 2024-12-04T08:25:57,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741883_1067 (size=15140) 2024-12-04T08:25:57,192 DEBUG [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/oldWALs 2024-12-04T08:25:57,192 INFO [RS:0;f5a5a857f5c5:37469 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C37469%2C1733300713386:(num 1733300751338) 2024-12-04T08:25:57,192 DEBUG [RS:0;f5a5a857f5c5:37469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:57,192 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:25:57,192 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:57,192 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:57,192 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:57,192 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:57,192 INFO [RS:0;f5a5a857f5c5:37469 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37469 2024-12-04T08:25:57,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,37469,1733300713386 2024-12-04T08:25:57,196 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:57,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:57,197 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,37469,1733300713386] 2024-12-04T08:25:57,198 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,37469,1733300713386 already deleted, retry=false 2024-12-04T08:25:57,198 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,37469,1733300713386 expired; onlineServers=0 2024-12-04T08:25:57,198 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,41417,1733300713326' ***** 2024-12-04T08:25:57,198 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:25:57,198 INFO [M:0;f5a5a857f5c5:41417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:25:57,198 INFO [M:0;f5a5a857f5c5:41417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:25:57,199 DEBUG [M:0;f5a5a857f5c5:41417 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:25:57,199 DEBUG [M:0;f5a5a857f5c5:41417 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:25:57,199 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:25:57,199 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300713656 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300713656,5,FailOnTimeoutGroup] 2024-12-04T08:25:57,199 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300713657 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300713657,5,FailOnTimeoutGroup] 2024-12-04T08:25:57,199 INFO [M:0;f5a5a857f5c5:41417 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:25:57,199 INFO [M:0;f5a5a857f5c5:41417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:25:57,199 DEBUG [M:0;f5a5a857f5c5:41417 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:25:57,199 INFO [M:0;f5a5a857f5c5:41417 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:25:57,199 INFO [M:0;f5a5a857f5c5:41417 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:25:57,200 INFO [M:0;f5a5a857f5c5:41417 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:25:57,200 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:25:57,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:25:57,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:57,200 DEBUG [M:0;f5a5a857f5c5:41417 {}] zookeeper.ZKUtil(347): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:25:57,200 WARN [M:0;f5a5a857f5c5:41417 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:25:57,201 INFO [M:0;f5a5a857f5c5:41417 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/.lastflushedseqids 2024-12-04T08:25:57,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741904_1090 (size=130) 2024-12-04T08:25:57,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741904_1090 (size=130) 2024-12-04T08:25:57,208 INFO [M:0;f5a5a857f5c5:41417 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:25:57,208 INFO [M:0;f5a5a857f5c5:41417 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:25:57,208 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:25:57,208 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:57,208 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:57,208 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:25:57,208 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:57,208 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB 2024-12-04T08:25:57,226 DEBUG [M:0;f5a5a857f5c5:41417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dff74ff395d4410692c0c8a2a4927021 is 82, key is hbase:meta,,1/info:regioninfo/1733300714317/Put/seqid=0 2024-12-04T08:25:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741905_1091 (size=5672) 2024-12-04T08:25:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741905_1091 (size=5672) 2024-12-04T08:25:57,232 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dff74ff395d4410692c0c8a2a4927021 2024-12-04T08:25:57,242 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:25:57,253 DEBUG [M:0;f5a5a857f5c5:41417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b704c6fa9a6c40f1bd027aed829b803a is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733300714928/Put/seqid=0 2024-12-04T08:25:57,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,260 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,260 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,260 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741906_1092 (size=6256) 2024-12-04T08:25:57,261 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741906_1092 (size=6256) 2024-12-04T08:25:57,262 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b704c6fa9a6c40f1bd027aed829b803a 2024-12-04T08:25:57,265 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,267 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:25:57,268 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b704c6fa9a6c40f1bd027aed829b803a 2024-12-04T08:25:57,282 DEBUG [M:0;f5a5a857f5c5:41417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/195d4ca321784b528060f8e5af804093 is 69, key is f5a5a857f5c5,35147,1733300714444/rs:state/1733300714493/Put/seqid=0 2024-12-04T08:25:57,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741907_1093 (size=5224) 2024-12-04T08:25:57,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741907_1093 (size=5224) 2024-12-04T08:25:57,288 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/195d4ca321784b528060f8e5af804093 2024-12-04T08:25:57,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:57,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37469-0x1017c93e1e80001, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:57,297 INFO [RS:0;f5a5a857f5c5:37469 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:57,297 INFO [RS:0;f5a5a857f5c5:37469 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,37469,1733300713386; zookeeper connection closed. 2024-12-04T08:25:57,297 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1322adf1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1322adf1 2024-12-04T08:25:57,298 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-04T08:25:57,307 DEBUG [M:0;f5a5a857f5c5:41417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/efccd66bee0c48cb9690a2c351594e02 is 52, key is load_balancer_on/state:d/1733300714420/Put/seqid=0 2024-12-04T08:25:57,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741908_1094 (size=5056) 2024-12-04T08:25:57,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741908_1094 (size=5056) 2024-12-04T08:25:57,313 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/efccd66bee0c48cb9690a2c351594e02 2024-12-04T08:25:57,319 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dff74ff395d4410692c0c8a2a4927021 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dff74ff395d4410692c0c8a2a4927021 2024-12-04T08:25:57,324 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dff74ff395d4410692c0c8a2a4927021, entries=8, sequenceid=60, filesize=5.5 K 2024-12-04T08:25:57,325 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b704c6fa9a6c40f1bd027aed829b803a as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b704c6fa9a6c40f1bd027aed829b803a 2024-12-04T08:25:57,330 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b704c6fa9a6c40f1bd027aed829b803a 2024-12-04T08:25:57,330 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b704c6fa9a6c40f1bd027aed829b803a, entries=6, sequenceid=60, filesize=6.1 K 2024-12-04T08:25:57,331 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/195d4ca321784b528060f8e5af804093 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/195d4ca321784b528060f8e5af804093 2024-12-04T08:25:57,337 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/195d4ca321784b528060f8e5af804093, entries=2, sequenceid=60, filesize=5.1 K 2024-12-04T08:25:57,338 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/efccd66bee0c48cb9690a2c351594e02 as hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/efccd66bee0c48cb9690a2c351594e02 2024-12-04T08:25:57,343 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/efccd66bee0c48cb9690a2c351594e02, entries=1, sequenceid=60, filesize=4.9 K 2024-12-04T08:25:57,345 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=60, compaction requested=false 2024-12-04T08:25:57,346 INFO [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:57,346 DEBUG [M:0;f5a5a857f5c5:41417 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300757208Disabling compacts and flushes for region at 1733300757208Disabling writes for close at 1733300757208Obtaining lock to block concurrent updates at 1733300757209 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300757209Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1733300757209Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300757210 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300757210Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300757225 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300757225Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300757237 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300757252 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300757252Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300757268 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300757282 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300757282Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300757293 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300757307 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300757307Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@359a537f: reopening flushed file at 1733300757318 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59486a94: reopening flushed file at 1733300757324 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e62034: reopening flushed file at 1733300757330 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40d1807d: reopening flushed file at 1733300757337 (+7 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=60, compaction requested=false at 1733300757345 (+8 ms)Writing region close event to WAL at 1733300757346 (+1 ms)Closed at 1733300757346 2024-12-04T08:25:57,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:25:57,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741879_1062 (size=1045) 2024-12-04T08:25:57,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37871 is added to blk_1073741879_1062 (size=1045) 2024-12-04T08:25:57,351 INFO [M:0;f5a5a857f5c5:41417 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:25:57,351 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:25:57,351 INFO [M:0;f5a5a857f5c5:41417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41417 2024-12-04T08:25:57,351 INFO [M:0;f5a5a857f5c5:41417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:25:57,455 INFO [M:0;f5a5a857f5c5:41417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:25:57,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:57,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41417-0x1017c93e1e80000, quorum=127.0.0.1:49602, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:25:57,459 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@719d00c0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:57,459 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@792fa80c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:57,459 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:57,459 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@743041f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:57,460 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c92a539{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:57,461 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:57,461 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:57,461 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6e14027b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:44007,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:45357 , LocalHost:localPort f5a5a857f5c5/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-04T08:25:57,461 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1202863867-172.17.0.2-1733300712426 (Datanode Uuid 65dac4eb-f1a3-451a-9b5f-769df3baa54a) service to localhost/127.0.0.1:36333 2024-12-04T08:25:57,461 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:57,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data3/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:57,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data4/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:57,462 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6e14027b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(293): Failed to updateBlock (newblock=BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1073, datanode=DatanodeInfoWithStorage[127.0.0.1:37871,null,null]) org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica does not exist BP-1202863867-172.17.0.2-1733300712426:1073741837 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getReplicaInfo(FsDatasetImpl.java:897) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.getStorageUuidForLock(FsDatasetImpl.java:905) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.updateReplicaUnderRecovery(FsDatasetImpl.java:3093) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode.updateReplicaUnderRecovery(DataNode.java:3537) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$BlockRecord.updateReplicaUnderRecovery(BlockRecoveryWorker.java:88) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$BlockRecord.access$700(BlockRecoveryWorker.java:71) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.syncBlock(BlockRecoveryWorker.java:289) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:183) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:57,462 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:57,462 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6e14027b {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37871,null,null], DatanodeInfoWithStorage[127.0.0.1:44007,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: Cannot recover BP-1202863867-172.17.0.2-1733300712426:blk_1073741837_1013, the following datanodes failed: [DatanodeInfoWithStorage[127.0.0.1:37871,null,null]] 2024-12-04T08:25:57,463 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6e14027b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37871,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1202863867-172.17.0.2-1733300712426 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:57,463 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6e14027b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:44007,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1202863867-172.17.0.2-1733300712426 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:25:57,463 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6e14027b {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37871,null,null], DatanodeInfoWithStorage[127.0.0.1:44007,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1202863867-172.17.0.2-1733300712426:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:37871,null,null], DatanodeInfoWithStorage[127.0.0.1:44007,null,null]] 2024-12-04T08:25:57,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ffc270{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:57,465 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2c1aeedb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:57,465 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:57,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c590722{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:57,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3986ff43{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:57,466 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:25:57,466 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:25:57,466 WARN [BP-1202863867-172.17.0.2-1733300712426 heartbeating to localhost/127.0.0.1:36333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1202863867-172.17.0.2-1733300712426 (Datanode Uuid 72a118e3-ab4e-4bd9-87d5-53dd9586738e) service to localhost/127.0.0.1:36333 2024-12-04T08:25:57,466 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:25:57,467 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data7/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:57,467 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/cluster_2a25f54a-3964-e605-bb8b-8ff1459146a7/data/data8/current/BP-1202863867-172.17.0.2-1733300712426 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:25:57,467 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:25:57,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6469263a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:25:57,473 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2324be38{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:25:57,473 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:25:57,473 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6abea1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:25:57,473 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d8a9c69{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir/,STOPPED} 2024-12-04T08:25:57,481 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:25:57,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:25:57,519 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=156 (was 81) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:45531 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36333 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36333 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45531 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36333 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36333 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36333 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:36333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36333 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$898/0x00007efeb4bf52a8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$898/0x00007efeb4bf52a8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36333 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36333 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:36333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:36333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=74 (was 115), ProcessCount=11 (was 11), AvailableMemoryMB=6593 (was 7328) 2024-12-04T08:25:57,527 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=156, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=74, ProcessCount=11, AvailableMemoryMB=6593 2024-12-04T08:25:57,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:25:57,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.log.dir so I do NOT create it in target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac 2024-12-04T08:25:57,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a53da31c-579c-6961-1370-b92276dd02aa/hadoop.tmp.dir so I do NOT create it in target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac 2024-12-04T08:25:57,527 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870, deleteOnExit=true 2024-12-04T08:25:57,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/test.cache.data in system properties and HBase conf 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:25:57,528 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:25:57,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:25:57,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:25:57,542 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:25:57,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:57,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:57,617 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:57,623 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:57,624 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:57,624 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:57,624 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:57,625 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:57,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10e56c5a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:57,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c23e5e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:57,739 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@228e200c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-33405-hadoop-hdfs-3_4_1-tests_jar-_-any-2534564126944521815/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:25:57,739 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3045a903{HTTP/1.1, (http/1.1)}{localhost:33405} 2024-12-04T08:25:57,739 INFO [Time-limited test {}] server.Server(415): Started @149153ms 2024-12-04T08:25:57,752 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:25:57,822 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:57,825 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:57,826 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:57,826 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:57,826 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:57,826 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444db7a8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:57,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77037455{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:57,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66c0323e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-43579-hadoop-hdfs-3_4_1-tests_jar-_-any-2168701510188581639/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:57,942 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d2191b9{HTTP/1.1, (http/1.1)}{localhost:43579} 2024-12-04T08:25:57,942 INFO [Time-limited test {}] server.Server(415): Started @149356ms 2024-12-04T08:25:57,944 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:57,974 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:25:57,977 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:25:57,978 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:25:57,978 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:25:57,978 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:25:57,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23fcbb95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:25:57,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6699fa8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:25:58,061 WARN [Thread-1192 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data2/current/BP-1952440342-172.17.0.2-1733300757560/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:58,061 WARN [Thread-1191 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data1/current/BP-1952440342-172.17.0.2-1733300757560/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:58,083 WARN [Thread-1170 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:58,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3cf55d75721ea588 with lease ID 0xc8b6fa4613d5def0: Processing first storage report for DS-e9242418-0089-4afd-a20a-4c38a4dca11a from datanode DatanodeRegistration(127.0.0.1:43205, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=43681, infoSecurePort=0, ipcPort=35289, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560) 2024-12-04T08:25:58,086 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3cf55d75721ea588 with lease ID 0xc8b6fa4613d5def0: from storage DS-e9242418-0089-4afd-a20a-4c38a4dca11a node DatanodeRegistration(127.0.0.1:43205, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=43681, infoSecurePort=0, ipcPort=35289, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:58,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3cf55d75721ea588 with lease ID 0xc8b6fa4613d5def0: Processing first storage report for DS-33972958-1e72-458d-879e-06b93e8a142c from datanode DatanodeRegistration(127.0.0.1:43205, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=43681, infoSecurePort=0, ipcPort=35289, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560) 2024-12-04T08:25:58,086 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3cf55d75721ea588 with lease ID 0xc8b6fa4613d5def0: from storage DS-33972958-1e72-458d-879e-06b93e8a142c node DatanodeRegistration(127.0.0.1:43205, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=43681, infoSecurePort=0, ipcPort=35289, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:58,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@33392a77{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-45633-hadoop-hdfs-3_4_1-tests_jar-_-any-5997518817885359698/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:25:58,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@57204301{HTTP/1.1, (http/1.1)}{localhost:45633} 2024-12-04T08:25:58,115 INFO [Time-limited test {}] server.Server(415): Started @149529ms 2024-12-04T08:25:58,117 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:25:58,211 WARN [Thread-1217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data3/current/BP-1952440342-172.17.0.2-1733300757560/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:58,211 WARN [Thread-1218 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data4/current/BP-1952440342-172.17.0.2-1733300757560/current, will proceed with Du for space computation calculation, 2024-12-04T08:25:58,227 WARN [Thread-1206 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:25:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf56f6bb2a513a704 with lease ID 0xc8b6fa4613d5def1: Processing first storage report for DS-f412bcb9-e607-4a49-9e47-9472243ce556 from datanode DatanodeRegistration(127.0.0.1:45861, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=46115, infoSecurePort=0, ipcPort=34705, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560) 2024-12-04T08:25:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf56f6bb2a513a704 with lease ID 0xc8b6fa4613d5def1: from storage DS-f412bcb9-e607-4a49-9e47-9472243ce556 node DatanodeRegistration(127.0.0.1:45861, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=46115, infoSecurePort=0, ipcPort=34705, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T08:25:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf56f6bb2a513a704 with lease ID 0xc8b6fa4613d5def1: Processing first storage report for DS-16a669b8-f519-4bbc-bd8b-599a1761c85c from datanode DatanodeRegistration(127.0.0.1:45861, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=46115, infoSecurePort=0, ipcPort=34705, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560) 2024-12-04T08:25:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf56f6bb2a513a704 with lease ID 0xc8b6fa4613d5def1: from storage DS-16a669b8-f519-4bbc-bd8b-599a1761c85c node DatanodeRegistration(127.0.0.1:45861, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=46115, infoSecurePort=0, ipcPort=34705, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:25:58,240 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac 2024-12-04T08:25:58,242 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/zookeeper_0, clientPort=58140, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:25:58,243 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58140 2024-12-04T08:25:58,244 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,245 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:25:58,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:25:58,254 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9 with version=8 2024-12-04T08:25:58,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:25:58,256 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:58,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:58,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:58,256 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:58,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:58,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:58,256 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:25:58,257 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:58,257 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36895 2024-12-04T08:25:58,259 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36895 connecting to ZooKeeper ensemble=127.0.0.1:58140 2024-12-04T08:25:58,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368950x0, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:58,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36895-0x1017c9491710000 connected 2024-12-04T08:25:58,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,281 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:58,283 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9, hbase.cluster.distributed=false 2024-12-04T08:25:58,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:58,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36895 2024-12-04T08:25:58,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36895 2024-12-04T08:25:58,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36895 2024-12-04T08:25:58,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36895 2024-12-04T08:25:58,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36895 2024-12-04T08:25:58,301 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:25:58,301 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:58,301 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:58,301 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:25:58,301 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:25:58,302 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:25:58,302 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:25:58,302 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:25:58,302 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45837 2024-12-04T08:25:58,303 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45837 connecting to ZooKeeper ensemble=127.0.0.1:58140 2024-12-04T08:25:58,304 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,306 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458370x0, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:25:58,310 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458370x0, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:25:58,310 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45837-0x1017c9491710001 connected 2024-12-04T08:25:58,310 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:25:58,310 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:25:58,311 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:25:58,312 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:25:58,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45837 2024-12-04T08:25:58,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45837 2024-12-04T08:25:58,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45837 2024-12-04T08:25:58,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45837 2024-12-04T08:25:58,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45837 2024-12-04T08:25:58,325 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:36895 2024-12-04T08:25:58,325 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:58,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:58,327 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:25:58,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,330 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:25:58,330 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,36895,1733300758256 from backup master directory 2024-12-04T08:25:58,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:58,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:25:58,331 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:58,331 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,336 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/hbase.id] with ID: cd057b64-c294-4cb9-a23c-92d1ee20b041 2024-12-04T08:25:58,336 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/.tmp/hbase.id 2024-12-04T08:25:58,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:25:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:25:58,342 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/.tmp/hbase.id]:[hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/hbase.id] 2024-12-04T08:25:58,354 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:58,354 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:25:58,355 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T08:25:58,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:58,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:25:58,363 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:25:58,364 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:25:58,364 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:58,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:25:58,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:25:58,371 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store 2024-12-04T08:25:58,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:58,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:25:58,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:58,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:25:58,378 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:58,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:58,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:25:58,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:58,378 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:25:58,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300758378Disabling compacts and flushes for region at 1733300758378Disabling writes for close at 1733300758378Writing region close event to WAL at 1733300758378Closed at 1733300758378 2024-12-04T08:25:58,379 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/.initializing 2024-12-04T08:25:58,379 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,381 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C36895%2C1733300758256, suffix=, logDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/oldWALs, maxLogs=10 2024-12-04T08:25:58,382 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 2024-12-04T08:25:58,386 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 2024-12-04T08:25:58,386 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43681:43681),(127.0.0.1/127.0.0.1:46115:46115)] 2024-12-04T08:25:58,387 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:58,387 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:58,387 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,387 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:25:58,390 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:58,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:25:58,392 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:58,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:25:58,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:58,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:25:58,395 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:58,396 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,397 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,397 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,398 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,398 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,399 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:25:58,400 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:25:58,402 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:58,402 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872679, jitterRate=0.10966919362545013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:25:58,403 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300758387Initializing all the Stores at 1733300758388 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300758388Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300758388Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300758388Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300758388Cleaning up temporary data from old regions at 1733300758398 (+10 ms)Region opened successfully at 1733300758402 (+4 ms) 2024-12-04T08:25:58,403 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:25:58,406 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1af84b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:58,407 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:25:58,407 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:25:58,407 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:25:58,407 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:25:58,407 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T08:25:58,408 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T08:25:58,408 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:25:58,410 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:25:58,411 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:25:58,412 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:25:58,412 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:25:58,413 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:25:58,415 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:25:58,415 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:25:58,416 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:25:58,417 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:25:58,418 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:25:58,419 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:25:58,421 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:25:58,422 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:25:58,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:58,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:25:58,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,429 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,36895,1733300758256, sessionid=0x1017c9491710000, setting cluster-up flag (Was=false) 2024-12-04T08:25:58,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,438 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:25:58,439 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,447 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:25:58,448 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:58,449 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:25:58,450 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:58,451 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:25:58,451 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:25:58,451 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,36895,1733300758256 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:58,452 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,453 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300788453 2024-12-04T08:25:58,453 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:25:58,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:25:58,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:25:58,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:25:58,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:25:58,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:25:58,454 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:58,454 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:25:58,454 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,455 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:25:58,455 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:25:58,455 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:25:58,455 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:25:58,455 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,455 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:25:58,455 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300758455,5,FailOnTimeoutGroup] 2024-12-04T08:25:58,455 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:25:58,456 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300758455,5,FailOnTimeoutGroup] 2024-12-04T08:25:58,456 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,456 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:25:58,456 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,456 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:25:58,466 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:25:58,466 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9 2024-12-04T08:25:58,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:25:58,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:25:58,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:58,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:25:58,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:25:58,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:58,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:25:58,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:25:58,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:58,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:25:58,482 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:25:58,482 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:58,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:25:58,484 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:25:58,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:58,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:58,484 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:25:58,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740 2024-12-04T08:25:58,485 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740 2024-12-04T08:25:58,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:25:58,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:25:58,487 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:25:58,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:25:58,490 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:58,491 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789731, jitterRate=0.004195675253868103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:25:58,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300758473Initializing all the Stores at 1733300758474 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300758474Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300758476 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300758476Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300758476Cleaning up temporary data from old regions at 1733300758487 (+11 ms)Region opened successfully at 1733300758491 (+4 ms) 2024-12-04T08:25:58,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:25:58,491 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:25:58,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:25:58,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:25:58,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:25:58,492 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:25:58,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300758491Disabling compacts and flushes for region at 1733300758491Disabling writes for close at 1733300758491Writing region close event to WAL at 1733300758492 (+1 ms)Closed at 1733300758492 2024-12-04T08:25:58,493 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:58,493 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:25:58,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:25:58,495 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:25:58,496 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:25:58,515 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(746): ClusterId : cd057b64-c294-4cb9-a23c-92d1ee20b041 2024-12-04T08:25:58,515 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:25:58,517 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:25:58,517 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:25:58,520 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:25:58,521 DEBUG [RS:0;f5a5a857f5c5:45837 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393661f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:25:58,533 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:45837 2024-12-04T08:25:58,533 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:25:58,533 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:25:58,533 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:25:58,534 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,36895,1733300758256 with port=45837, startcode=1733300758301 2024-12-04T08:25:58,534 DEBUG [RS:0;f5a5a857f5c5:45837 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:25:58,537 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35415, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:25:58,537 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36895 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,537 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36895 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,539 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9 2024-12-04T08:25:58,539 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38269 2024-12-04T08:25:58,539 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:25:58,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:25:58,542 DEBUG [RS:0;f5a5a857f5c5:45837 {}] zookeeper.ZKUtil(111): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,542 WARN [RS:0;f5a5a857f5c5:45837 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:25:58,542 INFO [RS:0;f5a5a857f5c5:45837 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:58,542 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,542 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,45837,1733300758301] 2024-12-04T08:25:58,545 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:25:58,547 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:25:58,547 INFO [RS:0;f5a5a857f5c5:45837 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:25:58,547 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,547 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:25:58,548 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:25:58,548 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,548 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:58,549 DEBUG [RS:0;f5a5a857f5c5:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:25:58,553 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,553 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,553 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,553 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,553 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,553 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,45837,1733300758301-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:58,568 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:25:58,568 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,45837,1733300758301-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,568 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,568 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.Replication(171): f5a5a857f5c5,45837,1733300758301 started 2024-12-04T08:25:58,582 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:58,582 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,45837,1733300758301, RpcServer on f5a5a857f5c5/172.17.0.2:45837, sessionid=0x1017c9491710001 2024-12-04T08:25:58,582 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:25:58,582 DEBUG [RS:0;f5a5a857f5c5:45837 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,582 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,45837,1733300758301' 2024-12-04T08:25:58,582 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,45837,1733300758301' 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:25:58,583 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:25:58,584 DEBUG [RS:0;f5a5a857f5c5:45837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:25:58,584 INFO [RS:0;f5a5a857f5c5:45837 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:25:58,584 INFO [RS:0;f5a5a857f5c5:45837 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:25:58,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:58,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:58,646 WARN [f5a5a857f5c5:36895 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:25:58,686 INFO [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C45837%2C1733300758301, suffix=, logDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs, maxLogs=32 2024-12-04T08:25:58,687 INFO [RS:0;f5a5a857f5c5:45837 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:25:58,693 INFO [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:25:58,694 DEBUG [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43681:43681),(127.0.0.1/127.0.0.1:46115:46115)] 2024-12-04T08:25:58,896 DEBUG [f5a5a857f5c5:36895 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:25:58,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:58,898 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,45837,1733300758301, state=OPENING 2024-12-04T08:25:58,900 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:25:58,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:25:58,902 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:25:58,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:58,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:58,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,45837,1733300758301}] 2024-12-04T08:25:59,055 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:25:59,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42791, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:25:59,061 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:25:59,061 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:25:59,063 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C45837%2C1733300758301.meta, suffix=.meta, logDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301, archiveDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs, maxLogs=32 2024-12-04T08:25:59,063 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta 2024-12-04T08:25:59,068 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta 2024-12-04T08:25:59,068 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43681:43681),(127.0.0.1/127.0.0.1:46115:46115)] 2024-12-04T08:25:59,069 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:59,069 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:25:59,069 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:25:59,070 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:25:59,070 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:25:59,070 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:59,070 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:25:59,070 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:25:59,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:25:59,072 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:25:59,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:59,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:59,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:25:59,073 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:25:59,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:59,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:59,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:25:59,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:25:59,075 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:59,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:59,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:25:59,076 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:25:59,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:59,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:25:59,076 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:25:59,077 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740 2024-12-04T08:25:59,078 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740 2024-12-04T08:25:59,079 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:25:59,079 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:25:59,080 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:25:59,081 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:25:59,082 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=705301, jitterRate=-0.10316382348537445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:25:59,082 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:25:59,082 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300759070Writing region info on filesystem at 1733300759070Initializing all the Stores at 1733300759071 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300759071Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300759071Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300759071Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300759071Cleaning up temporary data from old regions at 1733300759079 (+8 ms)Running coprocessor post-open hooks at 1733300759082 (+3 ms)Region opened successfully at 1733300759082 2024-12-04T08:25:59,083 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300759055 2024-12-04T08:25:59,086 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:25:59,086 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:25:59,087 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:59,088 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,45837,1733300758301, state=OPEN 2024-12-04T08:25:59,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:25:59,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:25:59,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:59,094 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:59,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:25:59,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:25:59,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,45837,1733300758301 in 192 msec 2024-12-04T08:25:59,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:25:59,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 603 msec 2024-12-04T08:25:59,100 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:25:59,100 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:25:59,101 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:25:59,102 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,45837,1733300758301, seqNum=-1] 2024-12-04T08:25:59,102 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:25:59,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:25:59,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 658 msec 2024-12-04T08:25:59,109 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300759109, completionTime=-1 2024-12-04T08:25:59,109 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:25:59,109 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300819111 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733300879111 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36895,1733300758256-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36895,1733300758256-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36895,1733300758256-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:36895, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,111 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,113 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.784sec 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36895,1733300758256-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:25:59,115 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36895,1733300758256-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:25:59,118 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:25:59,118 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:25:59,118 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36895,1733300758256-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:25:59,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cd82cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:59,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,36895,-1 for getting cluster id 2024-12-04T08:25:59,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:25:59,217 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd057b64-c294-4cb9-a23c-92d1ee20b041' 2024-12-04T08:25:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:25:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd057b64-c294-4cb9-a23c-92d1ee20b041" 2024-12-04T08:25:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2f0e9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,36895,-1] 2024-12-04T08:25:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:25:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:25:59,220 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:25:59,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277eda4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:25:59,221 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:25:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,45837,1733300758301, seqNum=-1] 2024-12-04T08:25:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:25:59,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:25:59,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:59,226 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:25:59,229 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:25:59,229 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-12-04T08:25:59,229 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-12-04T08:25:59,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T08:25:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is f5a5a857f5c5,36895,1733300758256 2024-12-04T08:25:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ed90965 2024-12-04T08:25:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T08:25:59,231 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T08:25:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T08:25:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T08:25:59,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:25:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T08:25:59,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T08:25:59,235 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:59,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-12-04T08:25:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:25:59,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T08:25:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741835_1011 (size=395) 2024-12-04T08:25:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741835_1011 (size=395) 2024-12-04T08:25:59,244 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d086d20f94d6a0b6a4d02ae468356fa9, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9 2024-12-04T08:25:59,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43205 is added to blk_1073741836_1012 (size=78) 2024-12-04T08:25:59,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45861 is added to blk_1073741836_1012 (size=78) 2024-12-04T08:25:59,252 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:59,252 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing d086d20f94d6a0b6a4d02ae468356fa9, disabling compactions & flushes 2024-12-04T08:25:59,252 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,252 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,252 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. after waiting 0 ms 2024-12-04T08:25:59,252 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,252 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,252 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for d086d20f94d6a0b6a4d02ae468356fa9: Waiting for close lock at 1733300759252Disabling compacts and flushes for region at 1733300759252Disabling writes for close at 1733300759252Writing region close event to WAL at 1733300759252Closed at 1733300759252 2024-12-04T08:25:59,254 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T08:25:59,254 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733300759254"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300759254"}]},"ts":"1733300759254"} 2024-12-04T08:25:59,256 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T08:25:59,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T08:25:59,257 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300759257"}]},"ts":"1733300759257"} 2024-12-04T08:25:59,259 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-12-04T08:25:59,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=d086d20f94d6a0b6a4d02ae468356fa9, ASSIGN}] 2024-12-04T08:25:59,261 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=d086d20f94d6a0b6a4d02ae468356fa9, ASSIGN 2024-12-04T08:25:59,262 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=d086d20f94d6a0b6a4d02ae468356fa9, ASSIGN; state=OFFLINE, location=f5a5a857f5c5,45837,1733300758301; forceNewPlan=false, retain=false 2024-12-04T08:25:59,412 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d086d20f94d6a0b6a4d02ae468356fa9, regionState=OPENING, regionLocation=f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:59,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=d086d20f94d6a0b6a4d02ae468356fa9, ASSIGN because future has completed 2024-12-04T08:25:59,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d086d20f94d6a0b6a4d02ae468356fa9, server=f5a5a857f5c5,45837,1733300758301}] 2024-12-04T08:25:59,573 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,573 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d086d20f94d6a0b6a4d02ae468356fa9, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:25:59,574 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,574 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:25:59,574 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,574 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,575 INFO [StoreOpener-d086d20f94d6a0b6a4d02ae468356fa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,577 INFO [StoreOpener-d086d20f94d6a0b6a4d02ae468356fa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d086d20f94d6a0b6a4d02ae468356fa9 columnFamilyName info 2024-12-04T08:25:59,577 DEBUG [StoreOpener-d086d20f94d6a0b6a4d02ae468356fa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:25:59,577 INFO [StoreOpener-d086d20f94d6a0b6a4d02ae468356fa9-1 {}] regionserver.HStore(327): Store=d086d20f94d6a0b6a4d02ae468356fa9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:25:59,577 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,578 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,578 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,579 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,579 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,580 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,582 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:25:59,583 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d086d20f94d6a0b6a4d02ae468356fa9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799863, jitterRate=0.017078831791877747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:25:59,583 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:25:59,584 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d086d20f94d6a0b6a4d02ae468356fa9: Running coprocessor pre-open hook at 1733300759574Writing region info on filesystem at 1733300759574Initializing all the Stores at 1733300759575 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300759575Cleaning up temporary data from old regions at 1733300759579 (+4 ms)Running coprocessor post-open hooks at 1733300759583 (+4 ms)Region opened successfully at 1733300759584 (+1 ms) 2024-12-04T08:25:59,585 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9., pid=6, masterSystemTime=1733300759569 2024-12-04T08:25:59,588 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,588 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:25:59,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d086d20f94d6a0b6a4d02ae468356fa9, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,45837,1733300758301 2024-12-04T08:25:59,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:59,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d086d20f94d6a0b6a4d02ae468356fa9, server=f5a5a857f5c5,45837,1733300758301 because future has completed 2024-12-04T08:25:59,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:25:59,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T08:25:59,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d086d20f94d6a0b6a4d02ae468356fa9, server=f5a5a857f5c5,45837,1733300758301 in 177 msec 2024-12-04T08:25:59,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T08:25:59,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=d086d20f94d6a0b6a4d02ae468356fa9, ASSIGN in 337 msec 2024-12-04T08:25:59,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T08:25:59,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300759600"}]},"ts":"1733300759600"} 2024-12-04T08:25:59,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-12-04T08:25:59,603 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T08:25:59,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 371 msec 2024-12-04T08:26:00,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:00,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:01,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:01,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:02,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:02,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:03,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:03,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:03,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T08:26:03,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T08:26:03,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T08:26:03,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-12-04T08:26:03,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:26:03,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T08:26:04,572 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:26:04,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,588 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,588 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,589 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,589 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,592 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,593 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,593 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:04,595 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:04,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:04,602 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T08:26:04,603 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-12-04T08:26:05,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:05,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:06,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:06,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:07,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:07,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:08,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:08,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:26:09,251 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-12-04T08:26:09,251 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-12-04T08:26:09,254 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T08:26:09,254 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:26:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9., hostname=f5a5a857f5c5,45837,1733300758301, seqNum=2] 2024-12-04T08:26:09,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:09,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:10,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:10,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:11,261 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:26:11,262 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:11,262 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:11,262 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:11,262 WARN [DataStreamer for file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 block BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK], DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]) is bad. 2024-12-04T08:26:11,262 WARN [DataStreamer for file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta block BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK], DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]) is bad. 2024-12-04T08:26:11,262 WARN [PacketResponder: BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45861] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,262 WARN [PacketResponder: BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45861] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,262 WARN [DataStreamer for file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 block BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK], DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45861,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]) is bad. 2024-12-04T08:26:11,263 WARN [PacketResponder: BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45861] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,263 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:54158 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54158 dst: /127.0.0.1:43205 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,263 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:54172 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54172 dst: /127.0.0.1:43205 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,263 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2075012943_22 at /127.0.0.1:54126 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54126 dst: /127.0.0.1:43205 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,263 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:49356 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49356 dst: /127.0.0.1:45861 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,263 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:49342 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49342 dst: /127.0.0.1:45861 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,264 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2075012943_22 at /127.0.0.1:49310 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49310 dst: /127.0.0.1:45861 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@33392a77{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:11,266 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@57204301{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:11,266 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:11,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6699fa8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:11,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23fcbb95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:11,268 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:26:11,268 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:26:11,268 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1952440342-172.17.0.2-1733300757560 (Datanode Uuid 935b419e-6444-4b60-8148-67bf5068a4b6) service to localhost/127.0.0.1:38269 2024-12-04T08:26:11,268 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:26:11,268 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data3/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:11,269 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data4/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:11,269 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:26:11,278 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:11,281 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:11,282 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:11,282 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:11,282 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:26:11,282 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4136ef12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:11,283 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36974255{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:11,400 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12f241e2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-46449-hadoop-hdfs-3_4_1-tests_jar-_-any-17312267140949598805/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:11,401 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2cce6536{HTTP/1.1, (http/1.1)}{localhost:46449} 2024-12-04T08:26:11,401 INFO [Time-limited test {}] server.Server(415): Started @162815ms 2024-12-04T08:26:11,403 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:26:11,425 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:11,425 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:11,425 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:11,426 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2075012943_22 at /127.0.0.1:44424 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44424 dst: /127.0.0.1:43205 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,426 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:44412 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44412 dst: /127.0.0.1:43205 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,426 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:44402 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44402 dst: /127.0.0.1:43205 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:11,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@66c0323e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:11,434 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d2191b9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:11,434 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:11,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77037455{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:11,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444db7a8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:11,435 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:26:11,436 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1952440342-172.17.0.2-1733300757560 (Datanode Uuid cce825da-1810-49d5-a289-17ef20c6583e) service to localhost/127.0.0.1:38269 2024-12-04T08:26:11,436 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:26:11,436 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:26:11,437 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data1/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:11,437 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data2/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:11,437 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:26:11,444 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:11,448 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:11,448 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:11,448 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:11,448 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:26:11,449 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11fd78ef{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:11,449 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43a7cd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:11,508 WARN [Thread-1341 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:26:11,510 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x67cde6cf14f70154 with lease ID 0xc8b6fa4613d5def2: from storage DS-f412bcb9-e607-4a49-9e47-9472243ce556 node DatanodeRegistration(127.0.0.1:36883, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=45191, infoSecurePort=0, ipcPort=40021, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:11,511 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x67cde6cf14f70154 with lease ID 0xc8b6fa4613d5def2: from storage DS-16a669b8-f519-4bbc-bd8b-599a1761c85c node DatanodeRegistration(127.0.0.1:36883, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=45191, infoSecurePort=0, ipcPort=40021, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:11,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:11,599 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f4861a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-35339-hadoop-hdfs-3_4_1-tests_jar-_-any-16968707037968567788/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:11,600 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68a8c2cd{HTTP/1.1, (http/1.1)}{localhost:35339} 2024-12-04T08:26:11,600 INFO [Time-limited test {}] server.Server(415): Started @163014ms 2024-12-04T08:26:11,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:11,601 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:26:11,696 WARN [Thread-1372 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:26:11,698 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3cd228ed63d43215 with lease ID 0xc8b6fa4613d5def3: from storage DS-e9242418-0089-4afd-a20a-4c38a4dca11a node DatanodeRegistration(127.0.0.1:36173, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=36877, infoSecurePort=0, ipcPort=38097, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:11,698 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3cd228ed63d43215 with lease ID 0xc8b6fa4613d5def3: from storage DS-33972958-1e72-458d-879e-06b93e8a142c node DatanodeRegistration(127.0.0.1:36173, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=36877, infoSecurePort=0, ipcPort=38097, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:12,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:12,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:12,620 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-12-04T08:26:12,622 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-12-04T08:26:12,624 ERROR [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:12,624 WARN [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:12,624 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C45837%2C1733300758301:(num 1733300758687) roll requested 2024-12-04T08:26:12,624 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:12,630 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 newFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:12,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:12,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:12,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:12,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:12,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:12,631 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:12,632 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:12,632 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:12,632 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:26:12,632 WARN [IPC Server handler 3 on default port 38269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013 2024-12-04T08:26:12,633 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 after 0ms 2024-12-04T08:26:12,637 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36877:36877),(127.0.0.1/127.0.0.1:45191:45191)] 2024-12-04T08:26:12,637 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 is not closed yet, will try archiving it next time 2024-12-04T08:26:13,510 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T08:26:13,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:13,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:14,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:14,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:14,640 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-12-04T08:26:15,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:15,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:16,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:16,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:16,633 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 after 4001ms 2024-12-04T08:26:16,643 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:16,643 WARN [DataStreamer for file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 block BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36173,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK], DatanodeInfoWithStorage[127.0.0.1:36883,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36173,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]) is bad. 2024-12-04T08:26:16,644 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:52400 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36173:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52400 dst: /127.0.0.1:36173 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:16,644 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:57450 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36883:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57450 dst: /127.0.0.1:36883 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:16,648 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f4861a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:16,648 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68a8c2cd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:16,648 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:16,648 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43a7cd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:16,649 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11fd78ef{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:16,650 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:26:16,650 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:26:16,650 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1952440342-172.17.0.2-1733300757560 (Datanode Uuid cce825da-1810-49d5-a289-17ef20c6583e) service to localhost/127.0.0.1:38269 2024-12-04T08:26:16,650 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:26:16,650 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data1/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:16,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data2/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:16,651 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:26:16,665 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:16,667 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:16,669 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:16,669 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:16,669 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:26:16,672 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f8a0d0d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:16,673 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57d5f4b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:16,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67d70e61{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-39283-hadoop-hdfs-3_4_1-tests_jar-_-any-9143462932683077907/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:16,786 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f1a012{HTTP/1.1, (http/1.1)}{localhost:39283} 2024-12-04T08:26:16,786 INFO [Time-limited test {}] server.Server(415): Started @168200ms 2024-12-04T08:26:16,788 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:26:16,804 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:16,805 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1235868741_22 at /127.0.0.1:57456 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36883:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57456 dst: /127.0.0.1:36883 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:16,809 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12f241e2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:16,810 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2cce6536{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:16,810 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:16,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36974255{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:16,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4136ef12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:16,811 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:26:16,811 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:26:16,811 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:26:16,811 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1952440342-172.17.0.2-1733300757560 (Datanode Uuid 935b419e-6444-4b60-8148-67bf5068a4b6) service to localhost/127.0.0.1:38269 2024-12-04T08:26:16,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data3/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:16,814 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data4/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:16,814 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:26:16,822 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:16,826 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:16,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:16,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:16,828 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:26:16,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49f94f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:16,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64685bd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:16,880 WARN [Thread-1415 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:26:16,882 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb4696a23650d095c with lease ID 0xc8b6fa4613d5def4: from storage DS-e9242418-0089-4afd-a20a-4c38a4dca11a node DatanodeRegistration(127.0.0.1:44957, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=35155, infoSecurePort=0, ipcPort=39881, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:16,882 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb4696a23650d095c with lease ID 0xc8b6fa4613d5def4: from storage DS-33972958-1e72-458d-879e-06b93e8a142c node DatanodeRegistration(127.0.0.1:44957, datanodeUuid=cce825da-1810-49d5-a289-17ef20c6583e, infoPort=35155, infoSecurePort=0, ipcPort=39881, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:16,946 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@652ca842{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/java.io.tmpdir/jetty-localhost-33213-hadoop-hdfs-3_4_1-tests_jar-_-any-5896140096650534723/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:16,947 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d802677{HTTP/1.1, (http/1.1)}{localhost:33213} 2024-12-04T08:26:16,947 INFO [Time-limited test {}] server.Server(415): Started @168360ms 2024-12-04T08:26:16,949 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:26:17,043 WARN [Thread-1446 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:26:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c6adbed1c7150a1 with lease ID 0xc8b6fa4613d5def5: from storage DS-f412bcb9-e607-4a49-9e47-9472243ce556 node DatanodeRegistration(127.0.0.1:46283, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=43603, infoSecurePort=0, ipcPort=38001, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c6adbed1c7150a1 with lease ID 0xc8b6fa4613d5def5: from storage DS-16a669b8-f519-4bbc-bd8b-599a1761c85c node DatanodeRegistration(127.0.0.1:46283, datanodeUuid=935b419e-6444-4b60-8148-67bf5068a4b6, infoPort=43603, infoSecurePort=0, ipcPort=38001, storageInfo=lv=-57;cid=testClusterID;nsid=1706185589;c=1733300757560), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:17,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:17,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:17,974 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-12-04T08:26:17,976 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-12-04T08:26:17,977 ERROR [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36883,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:17,977 WARN [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36883,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:17,978 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C45837%2C1733300758301:(num 1733300772624) roll requested 2024-12-04T08:26:17,978 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 2024-12-04T08:26:17,983 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 newFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 2024-12-04T08:26:17,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:17,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:17,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:17,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:17,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:17,984 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 2024-12-04T08:26:17,984 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36883,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:17,984 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36883,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:17,984 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:17,985 WARN [IPC Server handler 2 on default port 38269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-12-04T08:26:17,985 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 after 1ms 2024-12-04T08:26:17,988 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35155:35155),(127.0.0.1/127.0.0.1:43603:43603)] 2024-12-04T08:26:17,988 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 is not closed yet, will try archiving it next time 2024-12-04T08:26:18,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:18,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:18,884 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T08:26:19,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:19,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:19,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:19,995 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 newFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:19,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:19,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:19,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:19,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:19,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:19,996 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:19,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35155:35155),(127.0.0.1/127.0.0.1:43603:43603)] 2024-12-04T08:26:19,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 is not closed yet, will try archiving it next time 2024-12-04T08:26:19,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 is not closed yet, will try archiving it next time 2024-12-04T08:26:19,997 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:26:19,997 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:26:19,997 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 after 0ms 2024-12-04T08:26:19,998 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:26:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741838_1019 (size=1264) 2024-12-04T08:26:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741838_1019 (size=1264) 2024-12-04T08:26:19,999 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 is not closed yet, will try archiving it next time 2024-12-04T08:26:20,007 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733300759584/Put/vlen=218/seqid=0] 2024-12-04T08:26:20,007 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733300769259/Put/vlen=1045/seqid=0] 2024-12-04T08:26:20,007 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300758687 2024-12-04T08:26:20,007 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:20,007 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:20,007 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 after 0ms 2024-12-04T08:26:20,007 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:20,011 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733300772623/Put/vlen=1045/seqid=0] 2024-12-04T08:26:20,011 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733300774641/Put/vlen=1045/seqid=0] 2024-12-04T08:26:20,011 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 2024-12-04T08:26:20,011 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 2024-12-04T08:26:20,011 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 2024-12-04T08:26:20,011 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 after 0ms 2024-12-04T08:26:20,011 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300777978 2024-12-04T08:26:20,015 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733300777977/Put/vlen=1045/seqid=0] 2024-12-04T08:26:20,015 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:20,015 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:20,016 WARN [IPC Server handler 4 on default port 38269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-12-04T08:26:20,016 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 after 1ms 2024-12-04T08:26:20,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:20,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:20,884 WARN [ResponseProcessor for block BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:20,884 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2075012943_22 at /127.0.0.1:43460 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:44957:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43460 dst: /127.0.0.1:44957 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44957 remote=/127.0.0.1:43460]. Total timeout mills is 60000, 59111 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:20,884 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2075012943_22 at /127.0.0.1:53016 [Receiving block BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:46283:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53016 dst: /127.0.0.1:46283 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:20,884 WARN [DataStreamer for file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 block BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44957,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK], DatanodeInfoWithStorage[127.0.0.1:46283,DS-f412bcb9-e607-4a49-9e47-9472243ce556,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44957,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]) is bad. 2024-12-04T08:26:20,885 WARN [DataStreamer for file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 block BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:20,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741839_1022 (size=85) 2024-12-04T08:26:20,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741839_1022 (size=85) 2024-12-04T08:26:21,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:21,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:21,986 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300772624 after 4002ms 2024-12-04T08:26:22,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:22,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:23,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:23,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:24,017 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 after 4001ms 2024-12-04T08:26:24,017 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:24,020 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:24,021 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d086d20f94d6a0b6a4d02ae468356fa9 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-12-04T08:26:24,021 ERROR [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,021 WARN [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,022 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C45837%2C1733300758301:(num 1733300779990) roll requested 2024-12-04T08:26:24,022 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.1733300784022 2024-12-04T08:26:24,027 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 newFile=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300784022 2024-12-04T08:26:24,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,027 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300784022 2024-12-04T08:26:24,028 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,028 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1952440342-172.17.0.2-1733300757560:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,028 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:24,029 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35155:35155),(127.0.0.1/127.0.0.1:43603:43603)] 2024-12-04T08:26:24,029 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 after 1ms 2024-12-04T08:26:24,029 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 is not closed yet, will try archiving it next time 2024-12-04T08:26:24,029 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 to hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs/f5a5a857f5c5%2C45837%2C1733300758301.1733300779990 2024-12-04T08:26:24,045 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/.tmp/info/0ff534cac3c44744b11e5ad0fca4dd2a is 1080, key is row1002/info:/1733300769259/Put/seqid=0 2024-12-04T08:26:24,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741841_1024 (size=9270) 2024-12-04T08:26:24,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741841_1024 (size=9270) 2024-12-04T08:26:24,050 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/.tmp/info/0ff534cac3c44744b11e5ad0fca4dd2a 2024-12-04T08:26:24,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/.tmp/info/0ff534cac3c44744b11e5ad0fca4dd2a as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/info/0ff534cac3c44744b11e5ad0fca4dd2a 2024-12-04T08:26:24,062 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/info/0ff534cac3c44744b11e5ad0fca4dd2a, entries=4, sequenceid=8, filesize=9.1 K 2024-12-04T08:26:24,063 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for d086d20f94d6a0b6a4d02ae468356fa9 in 43ms, sequenceid=8, compaction requested=false 2024-12-04T08:26:24,063 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d086d20f94d6a0b6a4d02ae468356fa9: 2024-12-04T08:26:24,063 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-12-04T08:26:24,063 ERROR [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,063 WARN [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9-prefix:f5a5a857f5c5,45837,1733300758301.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,064 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C45837%2C1733300758301.meta:.meta(num 1733300759063) roll requested 2024-12-04T08:26:24,064 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300784064.meta 2024-12-04T08:26:24,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,071 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300784064.meta 2024-12-04T08:26:24,071 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,071 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:24,072 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta 2024-12-04T08:26:24,072 WARN [IPC Server handler 4 on default port 38269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014 2024-12-04T08:26:24,072 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta after 0ms 2024-12-04T08:26:24,072 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35155:35155),(127.0.0.1/127.0.0.1:43603:43603)] 2024-12-04T08:26:24,072 DEBUG [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta is not closed yet, will try archiving it next time 2024-12-04T08:26:24,087 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/info/9ae1dfea52ca4344aaf8084d5d5991dd is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9./info:regioninfo/1733300759589/Put/seqid=0 2024-12-04T08:26:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741843_1027 (size=7125) 2024-12-04T08:26:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741843_1027 (size=7125) 2024-12-04T08:26:24,092 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/info/9ae1dfea52ca4344aaf8084d5d5991dd 2024-12-04T08:26:24,111 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/ns/2155041759614764a549558800fda466 is 43, key is default/ns:d/1733300759104/Put/seqid=0 2024-12-04T08:26:24,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741844_1028 (size=5153) 2024-12-04T08:26:24,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741844_1028 (size=5153) 2024-12-04T08:26:24,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/ns/2155041759614764a549558800fda466 2024-12-04T08:26:24,136 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/table/2ca395ddc28340398ae4043fc5d9de4f is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733300759600/Put/seqid=0 2024-12-04T08:26:24,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741845_1029 (size=5438) 2024-12-04T08:26:24,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741845_1029 (size=5438) 2024-12-04T08:26:24,142 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/table/2ca395ddc28340398ae4043fc5d9de4f 2024-12-04T08:26:24,147 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/info/9ae1dfea52ca4344aaf8084d5d5991dd as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/info/9ae1dfea52ca4344aaf8084d5d5991dd 2024-12-04T08:26:24,152 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/info/9ae1dfea52ca4344aaf8084d5d5991dd, entries=10, sequenceid=11, filesize=7.0 K 2024-12-04T08:26:24,153 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/ns/2155041759614764a549558800fda466 as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/ns/2155041759614764a549558800fda466 2024-12-04T08:26:24,159 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/ns/2155041759614764a549558800fda466, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T08:26:24,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/.tmp/table/2ca395ddc28340398ae4043fc5d9de4f as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/table/2ca395ddc28340398ae4043fc5d9de4f 2024-12-04T08:26:24,165 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/table/2ca395ddc28340398ae4043fc5d9de4f, entries=2, sequenceid=11, filesize=5.3 K 2024-12-04T08:26:24,166 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=11, compaction requested=false 2024-12-04T08:26:24,166 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-04T08:26:24,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:26:24,171 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:26:24,171 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:26:24,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:26:24,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:26:24,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:26:24,171 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:26:24,171 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=924274750, stopped=false 2024-12-04T08:26:24,171 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,36895,1733300758256 2024-12-04T08:26:24,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:26:24,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:26:24,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:24,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:24,173 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:26:24,174 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:26:24,174 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:26:24,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:26:24,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:26:24,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:26:24,174 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,45837,1733300758301' ***** 2024-12-04T08:26:24,174 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:26:24,174 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:26:24,174 INFO [RS:0;f5a5a857f5c5:45837 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:26:24,174 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:26:24,174 INFO [RS:0;f5a5a857f5c5:45837 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:26:24,174 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(3091): Received CLOSE for d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,45837,1733300758301 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:45837. 2024-12-04T08:26:24,175 DEBUG [RS:0;f5a5a857f5c5:45837 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:26:24,175 DEBUG [RS:0;f5a5a857f5c5:45837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:26:24,175 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d086d20f94d6a0b6a4d02ae468356fa9, disabling compactions & flushes 2024-12-04T08:26:24,175 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:26:24,175 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:26:24,175 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. after waiting 0 ms 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:26:24,175 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:26:24,175 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T08:26:24,175 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1325): Online Regions={d086d20f94d6a0b6a4d02ae468356fa9=TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T08:26:24,175 DEBUG [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d086d20f94d6a0b6a4d02ae468356fa9 2024-12-04T08:26:24,175 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:26:24,175 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:26:24,175 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:26:24,176 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:26:24,176 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:26:24,179 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/default/TestLogRolling-testLogRollOnPipelineRestart/d086d20f94d6a0b6a4d02ae468356fa9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T08:26:24,180 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T08:26:24,180 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:26:24,180 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d086d20f94d6a0b6a4d02ae468356fa9: Waiting for close lock at 1733300784175Running coprocessor pre-close hooks at 1733300784175Disabling compacts and flushes for region at 1733300784175Disabling writes for close at 1733300784175Writing region close event to WAL at 1733300784176 (+1 ms)Running coprocessor post-close hooks at 1733300784180 (+4 ms)Closed at 1733300784180 2024-12-04T08:26:24,180 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:26:24,180 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733300759232.d086d20f94d6a0b6a4d02ae468356fa9. 2024-12-04T08:26:24,180 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:26:24,180 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300784175Running coprocessor pre-close hooks at 1733300784175Disabling compacts and flushes for region at 1733300784175Disabling writes for close at 1733300784176 (+1 ms)Writing region close event to WAL at 1733300784177 (+1 ms)Running coprocessor post-close hooks at 1733300784180 (+3 ms)Closed at 1733300784180 2024-12-04T08:26:24,180 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:26:24,375 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,45837,1733300758301; all regions closed. 2024-12-04T08:26:24,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:24,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741842_1025 (size=825) 2024-12-04T08:26:24,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741842_1025 (size=825) 2024-12-04T08:26:24,554 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:26:24,581 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T08:26:24,581 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T08:26:24,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:24,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:25,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:25,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:26,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:26,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:27,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:27,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:28,046 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T08:26:28,073 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta after 4001ms 2024-12-04T08:26:28,073 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/WALs/f5a5a857f5c5,45837,1733300758301/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta to hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs/f5a5a857f5c5%2C45837%2C1733300758301.meta.1733300759063.meta 2024-12-04T08:26:28,076 DEBUG [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs 2024-12-04T08:26:28,076 INFO [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C45837%2C1733300758301.meta:.meta(num 1733300784064) 2024-12-04T08:26:28,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741840_1023 (size=1162) 2024-12-04T08:26:28,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741840_1023 (size=1162) 2024-12-04T08:26:28,083 DEBUG [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs 2024-12-04T08:26:28,083 INFO [RS:0;f5a5a857f5c5:45837 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C45837%2C1733300758301:(num 1733300784022) 2024-12-04T08:26:28,083 DEBUG [RS:0;f5a5a857f5c5:45837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:26:28,083 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:26:28,083 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:26:28,083 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T08:26:28,083 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:26:28,083 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:26:28,083 INFO [RS:0;f5a5a857f5c5:45837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45837 2024-12-04T08:26:28,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,45837,1733300758301 2024-12-04T08:26:28,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:26:28,085 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:26:28,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,45837,1733300758301] 2024-12-04T08:26:28,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,45837,1733300758301 already deleted, retry=false 2024-12-04T08:26:28,089 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,45837,1733300758301 expired; onlineServers=0 2024-12-04T08:26:28,089 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,36895,1733300758256' ***** 2024-12-04T08:26:28,089 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:26:28,089 INFO [M:0;f5a5a857f5c5:36895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:26:28,089 INFO [M:0;f5a5a857f5c5:36895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:26:28,089 DEBUG [M:0;f5a5a857f5c5:36895 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:26:28,089 DEBUG [M:0;f5a5a857f5c5:36895 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:26:28,089 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:26:28,089 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300758455 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300758455,5,FailOnTimeoutGroup] 2024-12-04T08:26:28,089 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300758455 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300758455,5,FailOnTimeoutGroup] 2024-12-04T08:26:28,089 INFO [M:0;f5a5a857f5c5:36895 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:26:28,089 INFO [M:0;f5a5a857f5c5:36895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:26:28,090 DEBUG [M:0;f5a5a857f5c5:36895 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:26:28,090 INFO [M:0;f5a5a857f5c5:36895 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:26:28,090 INFO [M:0;f5a5a857f5c5:36895 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:26:28,090 INFO [M:0;f5a5a857f5c5:36895 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:26:28,090 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:26:28,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:26:28,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:28,091 DEBUG [M:0;f5a5a857f5c5:36895 {}] zookeeper.ZKUtil(347): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:26:28,091 WARN [M:0;f5a5a857f5c5:36895 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:26:28,091 INFO [M:0;f5a5a857f5c5:36895 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/.lastflushedseqids 2024-12-04T08:26:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741846_1030 (size=130) 2024-12-04T08:26:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741846_1030 (size=130) 2024-12-04T08:26:28,096 INFO [M:0;f5a5a857f5c5:36895 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:26:28,096 INFO [M:0;f5a5a857f5c5:36895 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:26:28,097 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:26:28,097 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:28,097 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:28,097 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:26:28,097 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:28,097 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB 2024-12-04T08:26:28,097 ERROR [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData-prefix:f5a5a857f5c5,36895,1733300758256 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:28,097 WARN [FSHLog-0-hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData-prefix:f5a5a857f5c5,36895,1733300758256 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:28,097 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog f5a5a857f5c5%2C36895%2C1733300758256:(num 1733300758381) roll requested 2024-12-04T08:26:28,098 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C36895%2C1733300758256.1733300788097 2024-12-04T08:26:28,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,103 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300788097 2024-12-04T08:26:28,104 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:28,104 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43205,DS-e9242418-0089-4afd-a20a-4c38a4dca11a,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-04T08:26:28,104 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 2024-12-04T08:26:28,104 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35155:35155),(127.0.0.1/127.0.0.1:43603:43603)] 2024-12-04T08:26:28,104 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 is not closed yet, will try archiving it next time 2024-12-04T08:26:28,104 WARN [IPC Server handler 2 on default port 38269 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015 2024-12-04T08:26:28,104 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 after 0ms 2024-12-04T08:26:28,119 DEBUG [M:0;f5a5a857f5c5:36895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5bc827c89c694efcbf5e00d54d71ea2c is 82, key is hbase:meta,,1/info:regioninfo/1733300759087/Put/seqid=0 2024-12-04T08:26:28,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741848_1033 (size=5672) 2024-12-04T08:26:28,125 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5bc827c89c694efcbf5e00d54d71ea2c 2024-12-04T08:26:28,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741848_1033 (size=5672) 2024-12-04T08:26:28,144 DEBUG [M:0;f5a5a857f5c5:36895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40b7b30548c748299783bb56ccb697db is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733300759605/Put/seqid=0 2024-12-04T08:26:28,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741849_1034 (size=6119) 2024-12-04T08:26:28,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741849_1034 (size=6119) 2024-12-04T08:26:28,149 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40b7b30548c748299783bb56ccb697db 2024-12-04T08:26:28,168 DEBUG [M:0;f5a5a857f5c5:36895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef8ec3ac11dc4b008592a3d0f1f1b1c2 is 69, key is f5a5a857f5c5,45837,1733300758301/rs:state/1733300758538/Put/seqid=0 2024-12-04T08:26:28,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741850_1035 (size=5156) 2024-12-04T08:26:28,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741850_1035 (size=5156) 2024-12-04T08:26:28,173 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef8ec3ac11dc4b008592a3d0f1f1b1c2 2024-12-04T08:26:28,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:26:28,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x1017c9491710001, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:26:28,188 INFO [RS:0;f5a5a857f5c5:45837 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:26:28,188 INFO [RS:0;f5a5a857f5c5:45837 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,45837,1733300758301; zookeeper connection closed. 2024-12-04T08:26:28,188 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d0d0082 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d0d0082 2024-12-04T08:26:28,188 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T08:26:28,191 DEBUG [M:0;f5a5a857f5c5:36895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4b4cc22ea25849408d369987363eefed is 52, key is load_balancer_on/state:d/1733300759227/Put/seqid=0 2024-12-04T08:26:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741851_1036 (size=5056) 2024-12-04T08:26:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741851_1036 (size=5056) 2024-12-04T08:26:28,196 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4b4cc22ea25849408d369987363eefed 2024-12-04T08:26:28,200 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5bc827c89c694efcbf5e00d54d71ea2c as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5bc827c89c694efcbf5e00d54d71ea2c 2024-12-04T08:26:28,204 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5bc827c89c694efcbf5e00d54d71ea2c, entries=8, sequenceid=56, filesize=5.5 K 2024-12-04T08:26:28,205 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40b7b30548c748299783bb56ccb697db as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/40b7b30548c748299783bb56ccb697db 2024-12-04T08:26:28,210 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/40b7b30548c748299783bb56ccb697db, entries=6, sequenceid=56, filesize=6.0 K 2024-12-04T08:26:28,210 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ef8ec3ac11dc4b008592a3d0f1f1b1c2 as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ef8ec3ac11dc4b008592a3d0f1f1b1c2 2024-12-04T08:26:28,215 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ef8ec3ac11dc4b008592a3d0f1f1b1c2, entries=1, sequenceid=56, filesize=5.0 K 2024-12-04T08:26:28,216 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4b4cc22ea25849408d369987363eefed as hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4b4cc22ea25849408d369987363eefed 2024-12-04T08:26:28,220 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4b4cc22ea25849408d369987363eefed, entries=1, sequenceid=56, filesize=4.9 K 2024-12-04T08:26:28,221 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=56, compaction requested=false 2024-12-04T08:26:28,223 INFO [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:28,223 DEBUG [M:0;f5a5a857f5c5:36895 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300788097Disabling compacts and flushes for region at 1733300788097Disabling writes for close at 1733300788097Obtaining lock to block concurrent updates at 1733300788097Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300788097Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733300788097Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300788105 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300788105Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300788119 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300788119Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300788130 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300788144 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300788144Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300788154 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300788167 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300788167Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300788177 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300788191 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300788191Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@314522b2: reopening flushed file at 1733300788199 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6012abf: reopening flushed file at 1733300788204 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72d12660: reopening flushed file at 1733300788210 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5281a0ae: reopening flushed file at 1733300788215 (+5 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=56, compaction requested=false at 1733300788221 (+6 ms)Writing region close event to WAL at 1733300788223 (+2 ms)Closed at 1733300788223 2024-12-04T08:26:28,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:26:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46283 is added to blk_1073741847_1031 (size=757) 2024-12-04T08:26:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741847_1031 (size=757) 2024-12-04T08:26:28,240 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:26:28,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:28,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:29,181 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,181 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,192 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,192 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,192 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,196 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,196 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,196 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,201 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,201 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:29,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:29,704 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:26:29,705 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,705 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,705 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,705 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,718 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,718 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,718 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,719 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,719 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,719 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,722 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,722 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,722 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:29,724 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:30,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:30,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:31,046 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-04T08:26:31,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:31,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:32,105 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 after 4001ms 2024-12-04T08:26:32,106 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/WALs/f5a5a857f5c5,36895,1733300758256/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 to hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/oldWALs/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 2024-12-04T08:26:32,109 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/MasterData/oldWALs/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381 to hdfs://localhost:38269/user/jenkins/test-data/bb0addfa-fa44-3417-0981-23d1cbd93fe9/oldWALs/f5a5a857f5c5%2C36895%2C1733300758256.1733300758381$masterlocalwal$ 2024-12-04T08:26:32,109 INFO [M:0;f5a5a857f5c5:36895 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:26:32,109 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:26:32,109 INFO [M:0;f5a5a857f5c5:36895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36895 2024-12-04T08:26:32,109 INFO [M:0;f5a5a857f5c5:36895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:26:32,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:26:32,211 INFO [M:0;f5a5a857f5c5:36895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:26:32,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36895-0x1017c9491710000, quorum=127.0.0.1:58140, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:26:32,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@652ca842{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:32,214 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d802677{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:32,214 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:32,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64685bd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:32,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49f94f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:32,216 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:26:32,216 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:26:32,216 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1952440342-172.17.0.2-1733300757560 (Datanode Uuid 935b419e-6444-4b60-8148-67bf5068a4b6) service to localhost/127.0.0.1:38269 2024-12-04T08:26:32,216 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:26:32,216 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data3/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:32,217 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data4/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:32,217 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:26:32,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67d70e61{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:32,219 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f1a012{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:32,219 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:32,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57d5f4b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:32,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f8a0d0d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:32,220 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:26:32,220 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:26:32,220 WARN [BP-1952440342-172.17.0.2-1733300757560 heartbeating to localhost/127.0.0.1:38269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1952440342-172.17.0.2-1733300757560 (Datanode Uuid cce825da-1810-49d5-a289-17ef20c6583e) service to localhost/127.0.0.1:38269 2024-12-04T08:26:32,220 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:26:32,221 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data1/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:32,221 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/cluster_a103e203-a9ed-9431-6501-c83d300a0870/data/data2/current/BP-1952440342-172.17.0.2-1733300757560 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:26:32,221 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:26:32,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@228e200c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:26:32,227 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3045a903{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:26:32,227 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:26:32,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c23e5e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:26:32,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10e56c5a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir/,STOPPED} 2024-12-04T08:26:32,233 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:26:32,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:26:32,259 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=181 (was 156) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38269 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38269 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:38269 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38269 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38269 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38269 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38269 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38269 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=41 (was 74), ProcessCount=11 (was 11), AvailableMemoryMB=6454 (was 6593) 2024-12-04T08:26:32,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=181, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=41, ProcessCount=11, AvailableMemoryMB=6454 2024-12-04T08:26:32,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:26:32,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.log.dir so I do NOT create it in target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e 2024-12-04T08:26:32,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c994842-af78-a063-359c-83e7bfa0aeac/hadoop.tmp.dir so I do NOT create it in target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e 2024-12-04T08:26:32,266 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d, deleteOnExit=true 2024-12-04T08:26:32,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/test.cache.data in system properties and HBase conf 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:26:32,267 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:26:32,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:26:32,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:26:32,281 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:26:32,353 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:32,357 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:32,358 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:32,358 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:32,358 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:26:32,359 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:32,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4761886e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:32,360 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70e475c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:32,473 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a31a089{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/java.io.tmpdir/jetty-localhost-40775-hadoop-hdfs-3_4_1-tests_jar-_-any-15430982680487180398/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:26:32,474 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d6dc4f9{HTTP/1.1, (http/1.1)}{localhost:40775} 2024-12-04T08:26:32,474 INFO [Time-limited test {}] server.Server(415): Started @183888ms 2024-12-04T08:26:32,487 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:26:32,537 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:32,540 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:32,541 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:32,541 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:32,541 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:26:32,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47ce5971{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:32,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e6ee234{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:32,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:32,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:32,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d0e51f4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/java.io.tmpdir/jetty-localhost-46475-hadoop-hdfs-3_4_1-tests_jar-_-any-6203094369881749386/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:32,661 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5838a3fd{HTTP/1.1, (http/1.1)}{localhost:46475} 2024-12-04T08:26:32,661 INFO [Time-limited test {}] server.Server(415): Started @184075ms 2024-12-04T08:26:32,663 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:26:32,690 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:26:32,693 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:26:32,693 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:26:32,693 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:26:32,693 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:26:32,694 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3fdc15a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:26:32,694 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20f59884{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:26:32,755 WARN [Thread-1640 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data1/current/BP-606264546-172.17.0.2-1733300792298/current, will proceed with Du for space computation calculation, 2024-12-04T08:26:32,755 WARN [Thread-1641 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data2/current/BP-606264546-172.17.0.2-1733300792298/current, will proceed with Du for space computation calculation, 2024-12-04T08:26:32,777 WARN [Thread-1619 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:26:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf16e81f5d446ff04 with lease ID 0x6e5d2387d6307c67: Processing first storage report for DS-2c3bdd16-a758-43ad-95f6-2ba9e5e99904 from datanode DatanodeRegistration(127.0.0.1:38343, datanodeUuid=550f1697-10a5-45da-a8fa-e5908d29a82a, infoPort=45051, infoSecurePort=0, ipcPort=39591, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298) 2024-12-04T08:26:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf16e81f5d446ff04 with lease ID 0x6e5d2387d6307c67: from storage DS-2c3bdd16-a758-43ad-95f6-2ba9e5e99904 node DatanodeRegistration(127.0.0.1:38343, datanodeUuid=550f1697-10a5-45da-a8fa-e5908d29a82a, infoPort=45051, infoSecurePort=0, ipcPort=39591, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf16e81f5d446ff04 with lease ID 0x6e5d2387d6307c67: Processing first storage report for DS-40b3665e-a9bf-4d08-866f-2ed539757dcc from datanode DatanodeRegistration(127.0.0.1:38343, datanodeUuid=550f1697-10a5-45da-a8fa-e5908d29a82a, infoPort=45051, infoSecurePort=0, ipcPort=39591, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298) 2024-12-04T08:26:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf16e81f5d446ff04 with lease ID 0x6e5d2387d6307c67: from storage DS-40b3665e-a9bf-4d08-866f-2ed539757dcc node DatanodeRegistration(127.0.0.1:38343, datanodeUuid=550f1697-10a5-45da-a8fa-e5908d29a82a, infoPort=45051, infoSecurePort=0, ipcPort=39591, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:32,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@719add8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/java.io.tmpdir/jetty-localhost-36745-hadoop-hdfs-3_4_1-tests_jar-_-any-3784145855878717375/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:26:32,810 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@df163d0{HTTP/1.1, (http/1.1)}{localhost:36745} 2024-12-04T08:26:32,810 INFO [Time-limited test {}] server.Server(415): Started @184224ms 2024-12-04T08:26:32,812 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:26:32,916 WARN [Thread-1666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data3/current/BP-606264546-172.17.0.2-1733300792298/current, will proceed with Du for space computation calculation, 2024-12-04T08:26:32,916 WARN [Thread-1667 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data4/current/BP-606264546-172.17.0.2-1733300792298/current, will proceed with Du for space computation calculation, 2024-12-04T08:26:32,939 WARN [Thread-1655 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:26:32,941 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa866d6eb42e58dea with lease ID 0x6e5d2387d6307c68: Processing first storage report for DS-b9a97080-3c15-4914-b74e-e7d5f8bf8734 from datanode DatanodeRegistration(127.0.0.1:46005, datanodeUuid=cff2aead-2f90-4257-8051-84a4f49421de, infoPort=33625, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298) 2024-12-04T08:26:32,941 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa866d6eb42e58dea with lease ID 0x6e5d2387d6307c68: from storage DS-b9a97080-3c15-4914-b74e-e7d5f8bf8734 node DatanodeRegistration(127.0.0.1:46005, datanodeUuid=cff2aead-2f90-4257-8051-84a4f49421de, infoPort=33625, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:32,941 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa866d6eb42e58dea with lease ID 0x6e5d2387d6307c68: Processing first storage report for DS-06545040-5da7-4871-827f-c5d8255cd584 from datanode DatanodeRegistration(127.0.0.1:46005, datanodeUuid=cff2aead-2f90-4257-8051-84a4f49421de, infoPort=33625, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298) 2024-12-04T08:26:32,941 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa866d6eb42e58dea with lease ID 0x6e5d2387d6307c68: from storage DS-06545040-5da7-4871-827f-c5d8255cd584 node DatanodeRegistration(127.0.0.1:46005, datanodeUuid=cff2aead-2f90-4257-8051-84a4f49421de, infoPort=33625, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=733624362;c=1733300792298), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:26:33,035 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e 2024-12-04T08:26:33,037 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/zookeeper_0, clientPort=50403, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:26:33,038 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50403 2024-12-04T08:26:33,038 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,040 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:26:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:26:33,051 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7 with version=8 2024-12-04T08:26:33,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:26:33,053 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:26:33,053 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:26:33,054 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38701 2024-12-04T08:26:33,055 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38701 connecting to ZooKeeper ensemble=127.0.0.1:50403 2024-12-04T08:26:33,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387010x0, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:26:33,064 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38701-0x1017c95195c0000 connected 2024-12-04T08:26:33,097 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,100 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:26:33,100 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7, hbase.cluster.distributed=false 2024-12-04T08:26:33,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:26:33,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38701 2024-12-04T08:26:33,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38701 2024-12-04T08:26:33,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38701 2024-12-04T08:26:33,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38701 2024-12-04T08:26:33,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38701 2024-12-04T08:26:33,119 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:26:33,119 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:26:33,120 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39299 2024-12-04T08:26:33,121 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39299 connecting to ZooKeeper ensemble=127.0.0.1:50403 2024-12-04T08:26:33,121 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,123 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392990x0, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:26:33,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:392990x0, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:26:33,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39299-0x1017c95195c0001 connected 2024-12-04T08:26:33,129 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:26:33,130 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:26:33,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:26:33,131 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:26:33,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39299 2024-12-04T08:26:33,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39299 2024-12-04T08:26:33,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39299 2024-12-04T08:26:33,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39299 2024-12-04T08:26:33,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39299 2024-12-04T08:26:33,144 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:38701 2024-12-04T08:26:33,144 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:26:33,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:26:33,146 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:26:33,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,148 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:26:33,148 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,38701,1733300793052 from backup master directory 2024-12-04T08:26:33,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:26:33,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:26:33,150 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:26:33,150 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,154 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/hbase.id] with ID: 4f2eb22c-df11-40c3-907b-ef73311fe112 2024-12-04T08:26:33,154 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/.tmp/hbase.id 2024-12-04T08:26:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:26:33,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:26:33,160 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/.tmp/hbase.id]:[hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/hbase.id] 2024-12-04T08:26:33,170 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,170 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:26:33,171 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T08:26:33,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:26:33,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:26:33,182 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:26:33,183 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:26:33,183 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:26:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:26:33,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:26:33,191 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store 2024-12-04T08:26:33,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:26:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:26:33,197 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:26:33,197 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:26:33,197 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:33,198 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:33,198 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:26:33,198 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:33,198 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:26:33,198 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300793197Disabling compacts and flushes for region at 1733300793197Disabling writes for close at 1733300793198 (+1 ms)Writing region close event to WAL at 1733300793198Closed at 1733300793198 2024-12-04T08:26:33,198 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/.initializing 2024-12-04T08:26:33,199 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/WALs/f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,201 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C38701%2C1733300793052, suffix=, logDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/WALs/f5a5a857f5c5,38701,1733300793052, archiveDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/oldWALs, maxLogs=10 2024-12-04T08:26:33,201 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C38701%2C1733300793052.1733300793201 2024-12-04T08:26:33,205 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/WALs/f5a5a857f5c5,38701,1733300793052/f5a5a857f5c5%2C38701%2C1733300793052.1733300793201 2024-12-04T08:26:33,206 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:33625:33625)] 2024-12-04T08:26:33,207 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:26:33,207 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:26:33,207 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,207 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,209 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:26:33,209 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:26:33,211 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:26:33,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:26:33,212 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:26:33,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:26:33,213 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:26:33,214 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,215 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,215 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,216 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,216 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,217 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:26:33,218 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:26:33,219 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:26:33,220 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=745635, jitterRate=-0.051876530051231384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:26:33,221 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300793207Initializing all the Stores at 1733300793208 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793208Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300793208Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300793208Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300793208Cleaning up temporary data from old regions at 1733300793216 (+8 ms)Region opened successfully at 1733300793220 (+4 ms) 2024-12-04T08:26:33,221 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:26:33,224 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ca5813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:26:33,224 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:26:33,225 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:26:33,225 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:26:33,225 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:26:33,225 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T08:26:33,225 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T08:26:33,226 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:26:33,227 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:26:33,228 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:26:33,230 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:26:33,230 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:26:33,231 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:26:33,232 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:26:33,232 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:26:33,233 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:26:33,234 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:26:33,235 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:26:33,237 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:26:33,239 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:26:33,241 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:26:33,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:26:33,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:26:33,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,243 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,38701,1733300793052, sessionid=0x1017c95195c0000, setting cluster-up flag (Was=false) 2024-12-04T08:26:33,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,251 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:26:33,252 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,263 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:26:33,264 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,265 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:26:33,266 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:26:33,267 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:26:33,267 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:26:33,267 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,38701,1733300793052 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:26:33,268 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300823269 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:26:33,269 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:26:33,270 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,270 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:26:33,270 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:26:33,270 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:26:33,270 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:26:33,270 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:26:33,270 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:26:33,270 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:26:33,270 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300793270,5,FailOnTimeoutGroup] 2024-12-04T08:26:33,271 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300793270,5,FailOnTimeoutGroup] 2024-12-04T08:26:33,271 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,271 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:26:33,271 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,271 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,271 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,271 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:26:33,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:26:33,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:26:33,278 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:26:33,278 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7 2024-12-04T08:26:33,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:26:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:26:33,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:26:33,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:26:33,286 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:26:33,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:26:33,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:26:33,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:26:33,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:26:33,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:26:33,291 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:26:33,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,291 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:26:33,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740 2024-12-04T08:26:33,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740 2024-12-04T08:26:33,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:26:33,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:26:33,294 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:26:33,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:26:33,297 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:26:33,297 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791784, jitterRate=0.006806328892707825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:26:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300793284Initializing all the Stores at 1733300793285 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793285Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793285Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300793285Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793285Cleaning up temporary data from old regions at 1733300793294 (+9 ms)Region opened successfully at 1733300793298 (+4 ms) 2024-12-04T08:26:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:26:33,298 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:26:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:26:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:26:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:26:33,298 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:26:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300793298Disabling compacts and flushes for region at 1733300793298Disabling writes for close at 1733300793298Writing region close event to WAL at 1733300793298Closed at 1733300793298 2024-12-04T08:26:33,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:26:33,299 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:26:33,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:26:33,301 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:26:33,302 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:26:33,334 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(746): ClusterId : 4f2eb22c-df11-40c3-907b-ef73311fe112 2024-12-04T08:26:33,334 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:26:33,337 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:26:33,337 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:26:33,339 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:26:33,339 DEBUG [RS:0;f5a5a857f5c5:39299 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c5a59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:26:33,351 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:39299 2024-12-04T08:26:33,351 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:26:33,351 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:26:33,351 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:26:33,352 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,38701,1733300793052 with port=39299, startcode=1733300793118 2024-12-04T08:26:33,352 DEBUG [RS:0;f5a5a857f5c5:39299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:26:33,354 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32963, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:26:33,354 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38701 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,354 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38701 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,356 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7 2024-12-04T08:26:33,356 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41949 2024-12-04T08:26:33,356 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:26:33,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:26:33,358 DEBUG [RS:0;f5a5a857f5c5:39299 {}] zookeeper.ZKUtil(111): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,358 WARN [RS:0;f5a5a857f5c5:39299 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:26:33,358 INFO [RS:0;f5a5a857f5c5:39299 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:26:33,358 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,359 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,39299,1733300793118] 2024-12-04T08:26:33,362 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:26:33,363 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:26:33,364 INFO [RS:0;f5a5a857f5c5:39299 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:26:33,364 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,364 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:26:33,364 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:26:33,365 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:26:33,365 DEBUG [RS:0;f5a5a857f5c5:39299 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:26:33,367 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,367 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,367 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,367 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,367 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,367 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,39299,1733300793118-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:26:33,382 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:26:33,382 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,39299,1733300793118-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,382 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,382 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.Replication(171): f5a5a857f5c5,39299,1733300793118 started 2024-12-04T08:26:33,395 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,395 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,39299,1733300793118, RpcServer on f5a5a857f5c5/172.17.0.2:39299, sessionid=0x1017c95195c0001 2024-12-04T08:26:33,395 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:26:33,395 DEBUG [RS:0;f5a5a857f5c5:39299 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,395 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,39299,1733300793118' 2024-12-04T08:26:33,395 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:26:33,396 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:26:33,396 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:26:33,396 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:26:33,396 DEBUG [RS:0;f5a5a857f5c5:39299 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,396 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,39299,1733300793118' 2024-12-04T08:26:33,396 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:26:33,397 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:26:33,397 DEBUG [RS:0;f5a5a857f5c5:39299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:26:33,397 INFO [RS:0;f5a5a857f5c5:39299 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:26:33,397 INFO [RS:0;f5a5a857f5c5:39299 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:26:33,452 WARN [f5a5a857f5c5:38701 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:26:33,499 INFO [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C39299%2C1733300793118, suffix=, logDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118, archiveDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/oldWALs, maxLogs=32 2024-12-04T08:26:33,499 INFO [RS:0;f5a5a857f5c5:39299 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C39299%2C1733300793118.1733300793499 2024-12-04T08:26:33,505 INFO [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300793499 2024-12-04T08:26:33,506 DEBUG [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33625:33625),(127.0.0.1/127.0.0.1:45051:45051)] 2024-12-04T08:26:33,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:33,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:33,702 DEBUG [f5a5a857f5c5:38701 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:26:33,703 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,704 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,39299,1733300793118, state=OPENING 2024-12-04T08:26:33,708 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:26:33,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:26:33,710 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:26:33,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:26:33,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:26:33,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,39299,1733300793118}] 2024-12-04T08:26:33,863 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:26:33,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37951, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:26:33,869 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:26:33,870 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:26:33,871 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C39299%2C1733300793118.meta, suffix=.meta, logDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118, archiveDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/oldWALs, maxLogs=32 2024-12-04T08:26:33,872 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C39299%2C1733300793118.meta.1733300793871.meta 2024-12-04T08:26:33,876 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.meta.1733300793871.meta 2024-12-04T08:26:33,883 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33625:33625),(127.0.0.1/127.0.0.1:45051:45051)] 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:26:33,886 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:26:33,886 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:26:33,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:26:33,889 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:26:33,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:26:33,890 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:26:33,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:26:33,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:26:33,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:26:33,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:26:33,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:26:33,893 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:26:33,893 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740 2024-12-04T08:26:33,894 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740 2024-12-04T08:26:33,895 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:26:33,895 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:26:33,895 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:26:33,896 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:26:33,897 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=828467, jitterRate=0.053451329469680786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:26:33,897 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:26:33,897 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300793887Writing region info on filesystem at 1733300793887Initializing all the Stores at 1733300793887Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793887Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793887Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300793887Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300793887Cleaning up temporary data from old regions at 1733300793895 (+8 ms)Running coprocessor post-open hooks at 1733300793897 (+2 ms)Region opened successfully at 1733300793897 2024-12-04T08:26:33,898 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300793863 2024-12-04T08:26:33,901 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:26:33,901 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:26:33,902 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,902 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,39299,1733300793118, state=OPEN 2024-12-04T08:26:33,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:26:33,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:26:33,908 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:33,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:26:33,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:26:33,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:26:33,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,39299,1733300793118 in 198 msec 2024-12-04T08:26:33,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:26:33,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 611 msec 2024-12-04T08:26:33,914 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:26:33,914 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:26:33,915 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:26:33,915 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,39299,1733300793118, seqNum=-1] 2024-12-04T08:26:33,916 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:26:33,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56525, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:26:33,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 656 msec 2024-12-04T08:26:33,923 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300793923, completionTime=-1 2024-12-04T08:26:33,923 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:26:33,923 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300853925 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733300913925 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,38701,1733300793052-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,38701,1733300793052-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,925 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,38701,1733300793052-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,926 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:38701, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,926 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,926 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,927 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.779sec 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,38701,1733300793052-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:26:33,929 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,38701,1733300793052-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:26:33,931 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:26:33,931 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:26:33,931 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,38701,1733300793052-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:26:33,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-04T08:26:33,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f411ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:26:33,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,38701,-1 for getting cluster id 2024-12-04T08:26:33,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:26:33,936 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4f2eb22c-df11-40c3-907b-ef73311fe112' 2024-12-04T08:26:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:26:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4f2eb22c-df11-40c3-907b-ef73311fe112" 2024-12-04T08:26:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282ffbbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:26:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,38701,-1] 2024-12-04T08:26:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:26:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:26:33,938 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:26:33,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5fe848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:26:33,940 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:26:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,39299,1733300793118, seqNum=-1] 2024-12-04T08:26:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:26:33,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:26:33,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,944 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:26:33,946 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:26:33,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T08:26:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is f5a5a857f5c5,38701,1733300793052 2024-12-04T08:26:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7874d486 2024-12-04T08:26:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T08:26:33,948 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T08:26:33,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T08:26:33,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T08:26:33,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:26:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:33,951 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T08:26:33,952 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:33,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-12-04T08:26:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:26:33,953 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T08:26:33,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741835_1011 (size=405) 2024-12-04T08:26:33,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741835_1011 (size=405) 2024-12-04T08:26:33,961 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ba39ce722c37eed80c39805c420b72d0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7 2024-12-04T08:26:33,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741836_1012 (size=88) 2024-12-04T08:26:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741836_1012 (size=88) 2024-12-04T08:26:33,967 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:26:33,967 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing ba39ce722c37eed80c39805c420b72d0, disabling compactions & flushes 2024-12-04T08:26:33,967 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:33,967 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:33,967 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. after waiting 0 ms 2024-12-04T08:26:33,967 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:33,967 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:33,968 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ba39ce722c37eed80c39805c420b72d0: Waiting for close lock at 1733300793967Disabling compacts and flushes for region at 1733300793967Disabling writes for close at 1733300793967Writing region close event to WAL at 1733300793967Closed at 1733300793967 2024-12-04T08:26:33,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T08:26:33,969 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733300793969"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300793969"}]},"ts":"1733300793969"} 2024-12-04T08:26:33,971 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T08:26:33,973 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T08:26:33,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300793973"}]},"ts":"1733300793973"} 2024-12-04T08:26:33,975 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-12-04T08:26:33,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ba39ce722c37eed80c39805c420b72d0, ASSIGN}] 2024-12-04T08:26:33,976 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ba39ce722c37eed80c39805c420b72d0, ASSIGN 2024-12-04T08:26:33,978 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ba39ce722c37eed80c39805c420b72d0, ASSIGN; state=OFFLINE, location=f5a5a857f5c5,39299,1733300793118; forceNewPlan=false, retain=false 2024-12-04T08:26:34,128 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ba39ce722c37eed80c39805c420b72d0, regionState=OPENING, regionLocation=f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:34,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ba39ce722c37eed80c39805c420b72d0, ASSIGN because future has completed 2024-12-04T08:26:34,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba39ce722c37eed80c39805c420b72d0, server=f5a5a857f5c5,39299,1733300793118}] 2024-12-04T08:26:34,288 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:34,288 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ba39ce722c37eed80c39805c420b72d0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:26:34,288 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,288 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:26:34,288 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,288 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,289 INFO [StoreOpener-ba39ce722c37eed80c39805c420b72d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,291 INFO [StoreOpener-ba39ce722c37eed80c39805c420b72d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba39ce722c37eed80c39805c420b72d0 columnFamilyName info 2024-12-04T08:26:34,291 DEBUG [StoreOpener-ba39ce722c37eed80c39805c420b72d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:26:34,291 INFO [StoreOpener-ba39ce722c37eed80c39805c420b72d0-1 {}] regionserver.HStore(327): Store=ba39ce722c37eed80c39805c420b72d0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:26:34,291 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,292 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,292 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,293 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,293 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,294 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,296 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:26:34,297 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ba39ce722c37eed80c39805c420b72d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=788714, jitterRate=0.002902492880821228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:26:34,297 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:26:34,298 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ba39ce722c37eed80c39805c420b72d0: Running coprocessor pre-open hook at 1733300794288Writing region info on filesystem at 1733300794288Initializing all the Stores at 1733300794289 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300794289Cleaning up temporary data from old regions at 1733300794293 (+4 ms)Running coprocessor post-open hooks at 1733300794297 (+4 ms)Region opened successfully at 1733300794298 (+1 ms) 2024-12-04T08:26:34,299 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0., pid=6, masterSystemTime=1733300794284 2024-12-04T08:26:34,301 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:34,301 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:34,302 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ba39ce722c37eed80c39805c420b72d0, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,39299,1733300793118 2024-12-04T08:26:34,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba39ce722c37eed80c39805c420b72d0, server=f5a5a857f5c5,39299,1733300793118 because future has completed 2024-12-04T08:26:34,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T08:26:34,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ba39ce722c37eed80c39805c420b72d0, server=f5a5a857f5c5,39299,1733300793118 in 174 msec 2024-12-04T08:26:34,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T08:26:34,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ba39ce722c37eed80c39805c420b72d0, ASSIGN in 333 msec 2024-12-04T08:26:34,312 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T08:26:34,312 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300794312"}]},"ts":"1733300794312"} 2024-12-04T08:26:34,314 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-12-04T08:26:34,315 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T08:26:34,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 366 msec 2024-12-04T08:26:34,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:34,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:35,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:35,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:36,613 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:36,613 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:37,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:37,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:38,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:38,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:39,388 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:26:39,389 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,389 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,389 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,390 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,390 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,390 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,404 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,404 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,405 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,405 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,405 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,406 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,408 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,409 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,409 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,411 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:26:39,417 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T08:26:39,417 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-12-04T08:26:39,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:39,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:40,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:40,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:41,616 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:41,616 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:42,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:42,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:43,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:43,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:43,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:43,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T08:26:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:26:43,981 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T08:26:43,982 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-12-04T08:26:43,985 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:43,985 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0., hostname=f5a5a857f5c5,39299,1733300793118, seqNum=2] 2024-12-04T08:26:43,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:44,000 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T08:26:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T08:26:44,002 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T08:26:44,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T08:26:44,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T08:26:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:44,162 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ba39ce722c37eed80c39805c420b72d0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T08:26:44,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/d4530a65029b4cb2acb5e40950b6a455 is 1080, key is row0001/info:/1733300803989/Put/seqid=0 2024-12-04T08:26:44,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741837_1013 (size=6033) 2024-12-04T08:26:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741837_1013 (size=6033) 2024-12-04T08:26:44,184 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/d4530a65029b4cb2acb5e40950b6a455 2024-12-04T08:26:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/d4530a65029b4cb2acb5e40950b6a455 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/d4530a65029b4cb2acb5e40950b6a455 2024-12-04T08:26:44,197 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/d4530a65029b4cb2acb5e40950b6a455, entries=1, sequenceid=5, filesize=5.9 K 2024-12-04T08:26:44,198 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ba39ce722c37eed80c39805c420b72d0 in 36ms, sequenceid=5, compaction requested=false 2024-12-04T08:26:44,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ba39ce722c37eed80c39805c420b72d0: 2024-12-04T08:26:44,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T08:26:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T08:26:44,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T08:26:44,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 200 msec 2024-12-04T08:26:44,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 211 msec 2024-12-04T08:26:44,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:44,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:45,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:45,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:46,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:46,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:47,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:47,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:48,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:48,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:49,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:49,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:50,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:50,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:51,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:51,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:52,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:52,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:53,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:53,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T08:26:54,031 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T08:26:54,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:26:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T08:26:54,037 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T08:26:54,038 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T08:26:54,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T08:26:54,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T08:26:54,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:54,191 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ba39ce722c37eed80c39805c420b72d0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T08:26:54,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/b6ea372edc78477ab89a46d8de16de80 is 1080, key is row0002/info:/1733300814032/Put/seqid=0 2024-12-04T08:26:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741838_1014 (size=6033) 2024-12-04T08:26:54,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741838_1014 (size=6033) 2024-12-04T08:26:54,201 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/b6ea372edc78477ab89a46d8de16de80 2024-12-04T08:26:54,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/b6ea372edc78477ab89a46d8de16de80 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/b6ea372edc78477ab89a46d8de16de80 2024-12-04T08:26:54,211 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/b6ea372edc78477ab89a46d8de16de80, entries=1, sequenceid=9, filesize=5.9 K 2024-12-04T08:26:54,212 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ba39ce722c37eed80c39805c420b72d0 in 21ms, sequenceid=9, compaction requested=false 2024-12-04T08:26:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ba39ce722c37eed80c39805c420b72d0: 2024-12-04T08:26:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:26:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T08:26:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T08:26:54,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T08:26:54,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 176 msec 2024-12-04T08:26:54,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec 2024-12-04T08:26:54,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:54,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:55,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:55,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:56,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:56,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:57,626 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:57,626 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:58,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:58,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:59,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:59,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:26:59,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 after 68047ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:26:59,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta after 68036ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T08:27:00,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:00,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:01,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:01,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:02,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:02,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:03,034 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:27:03,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:03,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T08:27:04,051 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T08:27:04,053 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C39299%2C1733300793118.1733300824053 2024-12-04T08:27:04,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:04,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:04,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:04,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:04,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:04,060 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300793499 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300824053 2024-12-04T08:27:04,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741833_1009 (size=5546) 2024-12-04T08:27:04,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741833_1009 (size=5546) 2024-12-04T08:27:04,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33625:33625),(127.0.0.1/127.0.0.1:45051:45051)] 2024-12-04T08:27:04,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300793499 is not closed yet, will try archiving it next time 2024-12-04T08:27:04,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:27:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:27:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T08:27:04,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T08:27:04,066 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T08:27:04,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T08:27:04,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T08:27:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:04,220 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ba39ce722c37eed80c39805c420b72d0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T08:27:04,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/3c971763ee4445e9bc8c70d7022a751c is 1080, key is row0003/info:/1733300824052/Put/seqid=0 2024-12-04T08:27:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741840_1016 (size=6033) 2024-12-04T08:27:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741840_1016 (size=6033) 2024-12-04T08:27:04,230 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/3c971763ee4445e9bc8c70d7022a751c 2024-12-04T08:27:04,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/3c971763ee4445e9bc8c70d7022a751c as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/3c971763ee4445e9bc8c70d7022a751c 2024-12-04T08:27:04,241 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/3c971763ee4445e9bc8c70d7022a751c, entries=1, sequenceid=13, filesize=5.9 K 2024-12-04T08:27:04,243 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ba39ce722c37eed80c39805c420b72d0 in 23ms, sequenceid=13, compaction requested=true 2024-12-04T08:27:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ba39ce722c37eed80c39805c420b72d0: 2024-12-04T08:27:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T08:27:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T08:27:04,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T08:27:04,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec 2024-12-04T08:27:04,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec 2024-12-04T08:27:04,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:04,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:05,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:05,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:06,632 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:06,632 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:07,633 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:07,633 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:08,633 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:08,633 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:09,634 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:09,634 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:10,634 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:10,634 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:11,635 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:11,635 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:12,636 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:12,636 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:13,636 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:13,636 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:13,932 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T08:27:13,932 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T08:27:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T08:27:14,131 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T08:27:14,131 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:27:14,132 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:27:14,132 DEBUG [Time-limited test {}] regionserver.HStore(1541): ba39ce722c37eed80c39805c420b72d0/info is initiating minor compaction (all files) 2024-12-04T08:27:14,133 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:27:14,133 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:14,133 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ba39ce722c37eed80c39805c420b72d0/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:14,133 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/d4530a65029b4cb2acb5e40950b6a455, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/b6ea372edc78477ab89a46d8de16de80, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/3c971763ee4445e9bc8c70d7022a751c] into tmpdir=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp, totalSize=17.7 K 2024-12-04T08:27:14,133 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d4530a65029b4cb2acb5e40950b6a455, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733300803989 2024-12-04T08:27:14,134 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b6ea372edc78477ab89a46d8de16de80, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733300814032 2024-12-04T08:27:14,134 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3c971763ee4445e9bc8c70d7022a751c, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733300824052 2024-12-04T08:27:14,145 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ba39ce722c37eed80c39805c420b72d0#info#compaction#44 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:27:14,145 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/df93546407fb4f8ab643992f081ed9a8 is 1080, key is row0001/info:/1733300803989/Put/seqid=0 2024-12-04T08:27:14,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741841_1017 (size=8296) 2024-12-04T08:27:14,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741841_1017 (size=8296) 2024-12-04T08:27:14,156 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/df93546407fb4f8ab643992f081ed9a8 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/df93546407fb4f8ab643992f081ed9a8 2024-12-04T08:27:14,163 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ba39ce722c37eed80c39805c420b72d0/info of ba39ce722c37eed80c39805c420b72d0 into df93546407fb4f8ab643992f081ed9a8(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:27:14,163 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ba39ce722c37eed80c39805c420b72d0: 2024-12-04T08:27:14,165 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C39299%2C1733300793118.1733300834165 2024-12-04T08:27:14,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:14,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:14,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:14,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:14,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:14,171 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300824053 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300834165 2024-12-04T08:27:14,172 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:33625:33625)] 2024-12-04T08:27:14,172 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300824053 is not closed yet, will try archiving it next time 2024-12-04T08:27:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741839_1015 (size=2520) 2024-12-04T08:27:14,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741839_1015 (size=2520) 2024-12-04T08:27:14,173 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300793499 to hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/oldWALs/f5a5a857f5c5%2C39299%2C1733300793118.1733300793499 2024-12-04T08:27:14,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:27:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:27:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T08:27:14,175 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-04T08:27:14,176 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T08:27:14,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T08:27:14,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T08:27:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:14,330 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ba39ce722c37eed80c39805c420b72d0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T08:27:14,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/0e37d286faa9438aaaea8694f572951f is 1080, key is row0000/info:/1733300834164/Put/seqid=0 2024-12-04T08:27:14,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741843_1019 (size=6033) 2024-12-04T08:27:14,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741843_1019 (size=6033) 2024-12-04T08:27:14,340 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/0e37d286faa9438aaaea8694f572951f 2024-12-04T08:27:14,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/0e37d286faa9438aaaea8694f572951f as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/0e37d286faa9438aaaea8694f572951f 2024-12-04T08:27:14,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/0e37d286faa9438aaaea8694f572951f, entries=1, sequenceid=18, filesize=5.9 K 2024-12-04T08:27:14,352 INFO [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ba39ce722c37eed80c39805c420b72d0 in 21ms, sequenceid=18, compaction requested=false 2024-12-04T08:27:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ba39ce722c37eed80c39805c420b72d0: 2024-12-04T08:27:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T08:27:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T08:27:14,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T08:27:14,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec 2024-12-04T08:27:14,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec 2024-12-04T08:27:14,637 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:14,637 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:15,637 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:15,637 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:16,638 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:16,638 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:17,639 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:17,639 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:18,639 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:18,639 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:19,288 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region ba39ce722c37eed80c39805c420b72d0, had cached 0 bytes from a total of 14329 2024-12-04T08:27:19,640 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:19,640 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:20,640 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:20,640 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:21,641 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:21,641 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:22,642 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:22,642 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:23,642 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:23,642 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T08:27:24,181 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-04T08:27:24,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C39299%2C1733300793118.1733300844184 2024-12-04T08:27:24,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,190 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300834165 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300844184 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:33625:33625)] 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300834165 is not closed yet, will try archiving it next time 2024-12-04T08:27:24,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:27:24,191 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/WALs/f5a5a857f5c5,39299,1733300793118/f5a5a857f5c5%2C39299%2C1733300793118.1733300824053 to hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/oldWALs/f5a5a857f5c5%2C39299%2C1733300793118.1733300824053 2024-12-04T08:27:24,191 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:27:24,191 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2126023107, stopped=false 2024-12-04T08:27:24,191 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,38701,1733300793052 2024-12-04T08:27:24,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:27:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741842_1018 (size=2026) 2024-12-04T08:27:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741842_1018 (size=2026) 2024-12-04T08:27:24,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:27:24,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:27:24,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:24,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:24,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:27:24,193 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:27:24,194 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:27:24,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:27:24,194 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,39299,1733300793118' ***** 2024-12-04T08:27:24,194 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:27:24,194 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:27:24,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:27:24,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:27:24,194 INFO [RS:0;f5a5a857f5c5:39299 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:27:24,194 INFO [RS:0;f5a5a857f5c5:39299 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:27:24,194 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:27:24,194 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(3091): Received CLOSE for ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:27:24,194 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,39299,1733300793118 2024-12-04T08:27:24,194 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:27:24,195 INFO [RS:0;f5a5a857f5c5:39299 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:39299. 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba39ce722c37eed80c39805c420b72d0, disabling compactions & flushes 2024-12-04T08:27:24,195 DEBUG [RS:0;f5a5a857f5c5:39299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:27:24,195 DEBUG [RS:0;f5a5a857f5c5:39299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:27:24,195 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. after waiting 0 ms 2024-12-04T08:27:24,195 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:27:24,195 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:24,195 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:27:24,195 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:27:24,195 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ba39ce722c37eed80c39805c420b72d0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-04T08:27:24,195 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T08:27:24,195 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1325): Online Regions={ba39ce722c37eed80c39805c420b72d0=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T08:27:24,195 DEBUG [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ba39ce722c37eed80c39805c420b72d0 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:27:24,195 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:27:24,195 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:27:24,195 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-12-04T08:27:24,199 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/5cd9aa8db75c471385d5b89de88ccb43 is 1080, key is row0001/info:/1733300844182/Put/seqid=0 2024-12-04T08:27:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741845_1021 (size=6033) 2024-12-04T08:27:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741845_1021 (size=6033) 2024-12-04T08:27:24,204 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/5cd9aa8db75c471385d5b89de88ccb43 2024-12-04T08:27:24,209 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/.tmp/info/5cd9aa8db75c471385d5b89de88ccb43 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/5cd9aa8db75c471385d5b89de88ccb43 2024-12-04T08:27:24,212 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/info/5e5680397d6a4b8ca00ea06d83e60956 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0./info:regioninfo/1733300794302/Put/seqid=0 2024-12-04T08:27:24,214 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/5cd9aa8db75c471385d5b89de88ccb43, entries=1, sequenceid=22, filesize=5.9 K 2024-12-04T08:27:24,216 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ba39ce722c37eed80c39805c420b72d0 in 21ms, sequenceid=22, compaction requested=true 2024-12-04T08:27:24,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741846_1022 (size=7308) 2024-12-04T08:27:24,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741846_1022 (size=7308) 2024-12-04T08:27:24,219 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/info/5e5680397d6a4b8ca00ea06d83e60956 2024-12-04T08:27:24,219 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/d4530a65029b4cb2acb5e40950b6a455, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/b6ea372edc78477ab89a46d8de16de80, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/3c971763ee4445e9bc8c70d7022a751c] to archive 2024-12-04T08:27:24,220 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T08:27:24,222 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/d4530a65029b4cb2acb5e40950b6a455 to hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/d4530a65029b4cb2acb5e40950b6a455 2024-12-04T08:27:24,223 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/b6ea372edc78477ab89a46d8de16de80 to hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/b6ea372edc78477ab89a46d8de16de80 2024-12-04T08:27:24,224 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/3c971763ee4445e9bc8c70d7022a751c to hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/info/3c971763ee4445e9bc8c70d7022a751c 2024-12-04T08:27:24,225 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f5a5a857f5c5:38701 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T08:27:24,225 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d4530a65029b4cb2acb5e40950b6a455=6033, b6ea372edc78477ab89a46d8de16de80=6033, 3c971763ee4445e9bc8c70d7022a751c=6033] 2024-12-04T08:27:24,231 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ba39ce722c37eed80c39805c420b72d0/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-12-04T08:27:24,231 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:24,231 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba39ce722c37eed80c39805c420b72d0: Waiting for close lock at 1733300844195Running coprocessor pre-close hooks at 1733300844195Disabling compacts and flushes for region at 1733300844195Disabling writes for close at 1733300844195Obtaining lock to block concurrent updates at 1733300844195Preparing flush snapshotting stores in ba39ce722c37eed80c39805c420b72d0 at 1733300844195Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733300844195Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. at 1733300844196 (+1 ms)Flushing ba39ce722c37eed80c39805c420b72d0/info: creating writer at 1733300844196Flushing ba39ce722c37eed80c39805c420b72d0/info: appending metadata at 1733300844198 (+2 ms)Flushing ba39ce722c37eed80c39805c420b72d0/info: closing flushed file at 1733300844198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ff5a77f: reopening flushed file at 1733300844209 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ba39ce722c37eed80c39805c420b72d0 in 21ms, sequenceid=22, compaction requested=true at 1733300844216 (+7 ms)Writing region close event to WAL at 1733300844228 (+12 ms)Running coprocessor post-close hooks at 1733300844231 (+3 ms)Closed at 1733300844231 2024-12-04T08:27:24,231 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733300793949.ba39ce722c37eed80c39805c420b72d0. 2024-12-04T08:27:24,237 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/ns/1ca937350efa4ccf8c5d073099abf3d6 is 43, key is default/ns:d/1733300793918/Put/seqid=0 2024-12-04T08:27:24,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741847_1023 (size=5153) 2024-12-04T08:27:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741847_1023 (size=5153) 2024-12-04T08:27:24,242 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/ns/1ca937350efa4ccf8c5d073099abf3d6 2024-12-04T08:27:24,260 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/table/bb67ce2051b94cd2a47e9dc3c1d42a4f is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733300794312/Put/seqid=0 2024-12-04T08:27:24,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741848_1024 (size=5508) 2024-12-04T08:27:24,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741848_1024 (size=5508) 2024-12-04T08:27:24,265 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/table/bb67ce2051b94cd2a47e9dc3c1d42a4f 2024-12-04T08:27:24,270 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/info/5e5680397d6a4b8ca00ea06d83e60956 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/info/5e5680397d6a4b8ca00ea06d83e60956 2024-12-04T08:27:24,274 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/info/5e5680397d6a4b8ca00ea06d83e60956, entries=10, sequenceid=11, filesize=7.1 K 2024-12-04T08:27:24,275 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/ns/1ca937350efa4ccf8c5d073099abf3d6 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/ns/1ca937350efa4ccf8c5d073099abf3d6 2024-12-04T08:27:24,280 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/ns/1ca937350efa4ccf8c5d073099abf3d6, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T08:27:24,280 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/.tmp/table/bb67ce2051b94cd2a47e9dc3c1d42a4f as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/table/bb67ce2051b94cd2a47e9dc3c1d42a4f 2024-12-04T08:27:24,285 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/table/bb67ce2051b94cd2a47e9dc3c1d42a4f, entries=2, sequenceid=11, filesize=5.4 K 2024-12-04T08:27:24,286 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false 2024-12-04T08:27:24,291 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T08:27:24,292 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:27:24,292 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:27:24,292 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300844195Running coprocessor pre-close hooks at 1733300844195Disabling compacts and flushes for region at 1733300844195Disabling writes for close at 1733300844195Obtaining lock to block concurrent updates at 1733300844195Preparing flush snapshotting stores in 1588230740 at 1733300844195Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733300844196 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733300844196Flushing 1588230740/info: creating writer at 1733300844196Flushing 1588230740/info: appending metadata at 1733300844212 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733300844212Flushing 1588230740/ns: creating writer at 1733300844223 (+11 ms)Flushing 1588230740/ns: appending metadata at 1733300844237 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733300844237Flushing 1588230740/table: creating writer at 1733300844247 (+10 ms)Flushing 1588230740/table: appending metadata at 1733300844260 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733300844260Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14f219c5: reopening flushed file at 1733300844269 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f4c08c8: reopening flushed file at 1733300844274 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21db161c: reopening flushed file at 1733300844280 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false at 1733300844286 (+6 ms)Writing region close event to WAL at 1733300844288 (+2 ms)Running coprocessor post-close hooks at 1733300844292 (+4 ms)Closed at 1733300844292 2024-12-04T08:27:24,292 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:27:24,367 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T08:27:24,367 INFO [regionserver/f5a5a857f5c5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T08:27:24,395 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,39299,1733300793118; all regions closed. 2024-12-04T08:27:24,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741834_1010 (size=3306) 2024-12-04T08:27:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741834_1010 (size=3306) 2024-12-04T08:27:24,400 DEBUG [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/oldWALs 2024-12-04T08:27:24,400 INFO [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C39299%2C1733300793118.meta:.meta(num 1733300793871) 2024-12-04T08:27:24,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741844_1020 (size=1252) 2024-12-04T08:27:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741844_1020 (size=1252) 2024-12-04T08:27:24,405 DEBUG [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/oldWALs 2024-12-04T08:27:24,405 INFO [RS:0;f5a5a857f5c5:39299 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C39299%2C1733300793118:(num 1733300844184) 2024-12-04T08:27:24,406 DEBUG [RS:0;f5a5a857f5c5:39299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:27:24,406 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:27:24,406 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:27:24,406 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T08:27:24,406 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:27:24,406 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:27:24,406 INFO [RS:0;f5a5a857f5c5:39299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39299 2024-12-04T08:27:24,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,39299,1733300793118 2024-12-04T08:27:24,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:27:24,408 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:27:24,409 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,39299,1733300793118] 2024-12-04T08:27:24,412 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,39299,1733300793118 already deleted, retry=false 2024-12-04T08:27:24,412 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,39299,1733300793118 expired; onlineServers=0 2024-12-04T08:27:24,412 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,38701,1733300793052' ***** 2024-12-04T08:27:24,412 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:27:24,412 INFO [M:0;f5a5a857f5c5:38701 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:27:24,412 INFO [M:0;f5a5a857f5c5:38701 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:27:24,413 DEBUG [M:0;f5a5a857f5c5:38701 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:27:24,413 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:27:24,413 DEBUG [M:0;f5a5a857f5c5:38701 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:27:24,413 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300793270 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300793270,5,FailOnTimeoutGroup] 2024-12-04T08:27:24,413 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300793270 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300793270,5,FailOnTimeoutGroup] 2024-12-04T08:27:24,413 INFO [M:0;f5a5a857f5c5:38701 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:27:24,413 INFO [M:0;f5a5a857f5c5:38701 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:27:24,413 DEBUG [M:0;f5a5a857f5c5:38701 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:27:24,413 INFO [M:0;f5a5a857f5c5:38701 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:27:24,413 INFO [M:0;f5a5a857f5c5:38701 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:27:24,413 INFO [M:0;f5a5a857f5c5:38701 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:27:24,413 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:27:24,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:27:24,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:24,414 DEBUG [M:0;f5a5a857f5c5:38701 {}] zookeeper.ZKUtil(347): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:27:24,414 WARN [M:0;f5a5a857f5c5:38701 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:27:24,415 INFO [M:0;f5a5a857f5c5:38701 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/.lastflushedseqids 2024-12-04T08:27:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741849_1025 (size=130) 2024-12-04T08:27:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741849_1025 (size=130) 2024-12-04T08:27:24,420 INFO [M:0;f5a5a857f5c5:38701 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:27:24,420 INFO [M:0;f5a5a857f5c5:38701 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:27:24,420 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:27:24,420 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:24,420 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:24,420 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:27:24,420 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:24,420 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.59 KB heapSize=55 KB 2024-12-04T08:27:24,436 DEBUG [M:0;f5a5a857f5c5:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/566796edf34f48e0b6e198bc5a7faeeb is 82, key is hbase:meta,,1/info:regioninfo/1733300793901/Put/seqid=0 2024-12-04T08:27:24,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741850_1026 (size=5672) 2024-12-04T08:27:24,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741850_1026 (size=5672) 2024-12-04T08:27:24,441 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/566796edf34f48e0b6e198bc5a7faeeb 2024-12-04T08:27:24,463 DEBUG [M:0;f5a5a857f5c5:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ee5a2cbbac0349f6bfd5cc6aba608606 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733300794316/Put/seqid=0 2024-12-04T08:27:24,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741851_1027 (size=7823) 2024-12-04T08:27:24,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741851_1027 (size=7823) 2024-12-04T08:27:24,469 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.99 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ee5a2cbbac0349f6bfd5cc6aba608606 2024-12-04T08:27:24,473 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ee5a2cbbac0349f6bfd5cc6aba608606 2024-12-04T08:27:24,487 DEBUG [M:0;f5a5a857f5c5:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/19f65ffb6fc9401db24801f026114755 is 69, key is f5a5a857f5c5,39299,1733300793118/rs:state/1733300793355/Put/seqid=0 2024-12-04T08:27:24,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741852_1028 (size=5156) 2024-12-04T08:27:24,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741852_1028 (size=5156) 2024-12-04T08:27:24,492 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/19f65ffb6fc9401db24801f026114755 2024-12-04T08:27:24,510 DEBUG [M:0;f5a5a857f5c5:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d62c8b3e0d44d26ad2a5127bfa229a5 is 52, key is load_balancer_on/state:d/1733300793945/Put/seqid=0 2024-12-04T08:27:24,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:27:24,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39299-0x1017c95195c0001, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:27:24,512 INFO [RS:0;f5a5a857f5c5:39299 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:27:24,512 INFO [RS:0;f5a5a857f5c5:39299 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,39299,1733300793118; zookeeper connection closed. 2024-12-04T08:27:24,512 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a49fa75 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a49fa75 2024-12-04T08:27:24,512 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T08:27:24,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741853_1029 (size=5056) 2024-12-04T08:27:24,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741853_1029 (size=5056) 2024-12-04T08:27:24,514 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d62c8b3e0d44d26ad2a5127bfa229a5 2024-12-04T08:27:24,519 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/566796edf34f48e0b6e198bc5a7faeeb as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/566796edf34f48e0b6e198bc5a7faeeb 2024-12-04T08:27:24,524 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/566796edf34f48e0b6e198bc5a7faeeb, entries=8, sequenceid=121, filesize=5.5 K 2024-12-04T08:27:24,525 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ee5a2cbbac0349f6bfd5cc6aba608606 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ee5a2cbbac0349f6bfd5cc6aba608606 2024-12-04T08:27:24,530 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ee5a2cbbac0349f6bfd5cc6aba608606 2024-12-04T08:27:24,530 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ee5a2cbbac0349f6bfd5cc6aba608606, entries=14, sequenceid=121, filesize=7.6 K 2024-12-04T08:27:24,531 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/19f65ffb6fc9401db24801f026114755 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/19f65ffb6fc9401db24801f026114755 2024-12-04T08:27:24,536 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/19f65ffb6fc9401db24801f026114755, entries=1, sequenceid=121, filesize=5.0 K 2024-12-04T08:27:24,536 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d62c8b3e0d44d26ad2a5127bfa229a5 as hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9d62c8b3e0d44d26ad2a5127bfa229a5 2024-12-04T08:27:24,541 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41949/user/jenkins/test-data/7d367f8f-fe6a-1b5f-649f-3749e902daa7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9d62c8b3e0d44d26ad2a5127bfa229a5, entries=1, sequenceid=121, filesize=4.9 K 2024-12-04T08:27:24,542 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.59 KB/44641, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=121, compaction requested=false 2024-12-04T08:27:24,543 INFO [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:24,543 DEBUG [M:0;f5a5a857f5c5:38701 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300844420Disabling compacts and flushes for region at 1733300844420Disabling writes for close at 1733300844420Obtaining lock to block concurrent updates at 1733300844420Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300844420Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44641, getHeapSize=56256, getOffHeapSize=0, getCellsCount=140 at 1733300844421 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300844421Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300844421Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300844435 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300844435Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300844446 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300844463 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300844463Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300844473 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300844487 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300844487Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300844496 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300844509 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300844509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b396ac: reopening flushed file at 1733300844519 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f7bcc1: reopening flushed file at 1733300844525 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ffca997: reopening flushed file at 1733300844530 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72aa2ef8: reopening flushed file at 1733300844536 (+6 ms)Finished flush of dataSize ~43.59 KB/44641, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=121, compaction requested=false at 1733300844542 (+6 ms)Writing region close event to WAL at 1733300844543 (+1 ms)Closed at 1733300844543 2024-12-04T08:27:24,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:27:24,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741830_1006 (size=53038) 2024-12-04T08:27:24,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38343 is added to blk_1073741830_1006 (size=53038) 2024-12-04T08:27:24,546 INFO [M:0;f5a5a857f5c5:38701 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:27:24,546 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:27:24,546 INFO [M:0;f5a5a857f5c5:38701 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38701 2024-12-04T08:27:24,547 INFO [M:0;f5a5a857f5c5:38701 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:27:24,643 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:24,643 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:24,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:27:24,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1017c95195c0000, quorum=127.0.0.1:50403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:27:24,648 INFO [M:0;f5a5a857f5c5:38701 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:27:24,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@719add8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:27:24,651 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@df163d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:27:24,651 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:27:24,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20f59884{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:27:24,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3fdc15a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir/,STOPPED} 2024-12-04T08:27:24,653 WARN [BP-606264546-172.17.0.2-1733300792298 heartbeating to localhost/127.0.0.1:41949 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:27:24,653 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:27:24,653 WARN [BP-606264546-172.17.0.2-1733300792298 heartbeating to localhost/127.0.0.1:41949 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-606264546-172.17.0.2-1733300792298 (Datanode Uuid cff2aead-2f90-4257-8051-84a4f49421de) service to localhost/127.0.0.1:41949 2024-12-04T08:27:24,653 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:27:24,653 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data3/current/BP-606264546-172.17.0.2-1733300792298 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:27:24,654 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data4/current/BP-606264546-172.17.0.2-1733300792298 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:27:24,654 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:27:24,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d0e51f4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:27:24,657 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5838a3fd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:27:24,657 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:27:24,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e6ee234{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:27:24,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47ce5971{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir/,STOPPED} 2024-12-04T08:27:24,658 WARN [BP-606264546-172.17.0.2-1733300792298 heartbeating to localhost/127.0.0.1:41949 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:27:24,658 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:27:24,658 WARN [BP-606264546-172.17.0.2-1733300792298 heartbeating to localhost/127.0.0.1:41949 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-606264546-172.17.0.2-1733300792298 (Datanode Uuid 550f1697-10a5-45da-a8fa-e5908d29a82a) service to localhost/127.0.0.1:41949 2024-12-04T08:27:24,658 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:27:24,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data1/current/BP-606264546-172.17.0.2-1733300792298 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:27:24,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/cluster_3823c8e2-457f-d6f7-495c-b6e7e463075d/data/data2/current/BP-606264546-172.17.0.2-1733300792298 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:27:24,659 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:27:24,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a31a089{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:27:24,666 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d6dc4f9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:27:24,666 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:27:24,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70e475c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:27:24,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4761886e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir/,STOPPED} 2024-12-04T08:27:24,673 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:27:24,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:27:24,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=207 (was 181) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41949 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41949 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41949 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:41949 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f5a5a857f5c5:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41949 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41949 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41949 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41949 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=16 (was 41), ProcessCount=11 (was 11), AvailableMemoryMB=6405 (was 6454) 2024-12-04T08:27:24,706 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=207, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=16, ProcessCount=11, AvailableMemoryMB=6406 2024-12-04T08:27:24,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:27:24,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.log.dir so I do NOT create it in target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1 2024-12-04T08:27:24,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e6b155b-c980-6174-e427-5a3abdf0d35e/hadoop.tmp.dir so I do NOT create it in target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1 2024-12-04T08:27:24,706 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738, deleteOnExit=true 2024-12-04T08:27:24,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/test.cache.data in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:27:24,707 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:27:24,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:27:24,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:27:24,721 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:27:24,778 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:27:24,781 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:27:24,782 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:27:24,782 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:27:24,782 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T08:27:24,783 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:27:24,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@240fc28c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:27:24,784 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@314bb674{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:27:24,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25570184{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/java.io.tmpdir/jetty-localhost-39873-hadoop-hdfs-3_4_1-tests_jar-_-any-12881880833318929929/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:27:24,900 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@148b0695{HTTP/1.1, (http/1.1)}{localhost:39873} 2024-12-04T08:27:24,900 INFO [Time-limited test {}] server.Server(415): Started @236314ms 2024-12-04T08:27:24,914 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:27:25,011 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:27:25,013 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:27:25,021 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:27:25,021 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:27:25,021 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:27:25,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d639fc0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:27:25,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50f209dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:27:25,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5411f427{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/java.io.tmpdir/jetty-localhost-33383-hadoop-hdfs-3_4_1-tests_jar-_-any-11023469760472979241/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:27:25,150 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10e59f6e{HTTP/1.1, (http/1.1)}{localhost:33383} 2024-12-04T08:27:25,150 INFO [Time-limited test {}] server.Server(415): Started @236564ms 2024-12-04T08:27:25,151 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:27:25,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:27:25,183 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:27:25,183 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:27:25,183 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:27:25,184 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:27:25,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@314e7370{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:27:25,186 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@425d5d71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:27:25,239 WARN [Thread-1957 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data1/current/BP-1748760397-172.17.0.2-1733300844727/current, will proceed with Du for space computation calculation, 2024-12-04T08:27:25,239 WARN [Thread-1958 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data2/current/BP-1748760397-172.17.0.2-1733300844727/current, will proceed with Du for space computation calculation, 2024-12-04T08:27:25,255 WARN [Thread-1936 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:27:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x758d0d1976e5904e with lease ID 0x1072b3030d537181: Processing first storage report for DS-f68d7831-0f02-4b96-8e26-05522ef9e52f from datanode DatanodeRegistration(127.0.0.1:35299, datanodeUuid=34e809a6-b925-43e6-ae10-ee8f2ada7b91, infoPort=44017, infoSecurePort=0, ipcPort=39285, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727) 2024-12-04T08:27:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x758d0d1976e5904e with lease ID 0x1072b3030d537181: from storage DS-f68d7831-0f02-4b96-8e26-05522ef9e52f node DatanodeRegistration(127.0.0.1:35299, datanodeUuid=34e809a6-b925-43e6-ae10-ee8f2ada7b91, infoPort=44017, infoSecurePort=0, ipcPort=39285, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:27:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x758d0d1976e5904e with lease ID 0x1072b3030d537181: Processing first storage report for DS-920a2f34-c4db-4f49-81b4-2b0a451c2567 from datanode DatanodeRegistration(127.0.0.1:35299, datanodeUuid=34e809a6-b925-43e6-ae10-ee8f2ada7b91, infoPort=44017, infoSecurePort=0, ipcPort=39285, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727) 2024-12-04T08:27:25,257 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x758d0d1976e5904e with lease ID 0x1072b3030d537181: from storage DS-920a2f34-c4db-4f49-81b4-2b0a451c2567 node DatanodeRegistration(127.0.0.1:35299, datanodeUuid=34e809a6-b925-43e6-ae10-ee8f2ada7b91, infoPort=44017, infoSecurePort=0, ipcPort=39285, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:27:25,301 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d7cc900{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/java.io.tmpdir/jetty-localhost-44625-hadoop-hdfs-3_4_1-tests_jar-_-any-17320783206885913557/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:27:25,302 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20d96a0d{HTTP/1.1, (http/1.1)}{localhost:44625} 2024-12-04T08:27:25,302 INFO [Time-limited test {}] server.Server(415): Started @236716ms 2024-12-04T08:27:25,303 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:27:25,369 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:27:25,400 WARN [Thread-1983 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data3/current/BP-1748760397-172.17.0.2-1733300844727/current, will proceed with Du for space computation calculation, 2024-12-04T08:27:25,400 WARN [Thread-1984 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data4/current/BP-1748760397-172.17.0.2-1733300844727/current, will proceed with Du for space computation calculation, 2024-12-04T08:27:25,422 WARN [Thread-1972 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:27:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4f90f627002a01e with lease ID 0x1072b3030d537182: Processing first storage report for DS-9be16965-16b2-409a-add7-016a068044d9 from datanode DatanodeRegistration(127.0.0.1:44871, datanodeUuid=87c9e3e8-a639-4268-a848-52e83c69b4bd, infoPort=41683, infoSecurePort=0, ipcPort=40317, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727) 2024-12-04T08:27:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4f90f627002a01e with lease ID 0x1072b3030d537182: from storage DS-9be16965-16b2-409a-add7-016a068044d9 node DatanodeRegistration(127.0.0.1:44871, datanodeUuid=87c9e3e8-a639-4268-a848-52e83c69b4bd, infoPort=41683, infoSecurePort=0, ipcPort=40317, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:27:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4f90f627002a01e with lease ID 0x1072b3030d537182: Processing first storage report for DS-f1edefc7-4efb-421b-ad08-a555268d8c01 from datanode DatanodeRegistration(127.0.0.1:44871, datanodeUuid=87c9e3e8-a639-4268-a848-52e83c69b4bd, infoPort=41683, infoSecurePort=0, ipcPort=40317, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727) 2024-12-04T08:27:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4f90f627002a01e with lease ID 0x1072b3030d537182: from storage DS-f1edefc7-4efb-421b-ad08-a555268d8c01 node DatanodeRegistration(127.0.0.1:44871, datanodeUuid=87c9e3e8-a639-4268-a848-52e83c69b4bd, infoPort=41683, infoSecurePort=0, ipcPort=40317, storageInfo=lv=-57;cid=testClusterID;nsid=1432323896;c=1733300844727), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:27:25,524 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1 2024-12-04T08:27:25,528 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/zookeeper_0, clientPort=60685, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:27:25,529 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60685 2024-12-04T08:27:25,529 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:27:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:27:25,539 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3 with version=8 2024-12-04T08:27:25,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:27:25,541 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:27:25,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:27:25,544 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36131 2024-12-04T08:27:25,545 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36131 connecting to ZooKeeper ensemble=127.0.0.1:60685 2024-12-04T08:27:25,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361310x0, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:27:25,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36131-0x1017c95e6660000 connected 2024-12-04T08:27:25,568 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,569 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:27:25,571 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3, hbase.cluster.distributed=false 2024-12-04T08:27:25,572 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:27:25,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36131 2024-12-04T08:27:25,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36131 2024-12-04T08:27:25,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36131 2024-12-04T08:27:25,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36131 2024-12-04T08:27:25,574 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36131 2024-12-04T08:27:25,589 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:27:25,589 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:27:25,590 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42763 2024-12-04T08:27:25,591 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42763 connecting to ZooKeeper ensemble=127.0.0.1:60685 2024-12-04T08:27:25,591 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,593 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427630x0, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:27:25,597 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427630x0, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:27:25,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42763-0x1017c95e6660001 connected 2024-12-04T08:27:25,597 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:27:25,597 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:27:25,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:27:25,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:27:25,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42763 2024-12-04T08:27:25,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42763 2024-12-04T08:27:25,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42763 2024-12-04T08:27:25,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42763 2024-12-04T08:27:25,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42763 2024-12-04T08:27:25,611 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:36131 2024-12-04T08:27:25,612 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:27:25,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:27:25,615 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:27:25,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,616 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:27:25,617 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,36131,1733300845541 from backup master directory 2024-12-04T08:27:25,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:27:25,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:27:25,618 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:27:25,618 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,622 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/hbase.id] with ID: dceb7ae4-528d-4a19-a8ca-43c85c63f49c 2024-12-04T08:27:25,622 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/.tmp/hbase.id 2024-12-04T08:27:25,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:27:25,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:27:25,629 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/.tmp/hbase.id]:[hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/hbase.id] 2024-12-04T08:27:25,638 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:25,638 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:27:25,639 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T08:27:25,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,643 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:25,643 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:27:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:27:25,647 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:27:25,648 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:27:25,648 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:27:25,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:27:25,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:27:25,655 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store 2024-12-04T08:27:25,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:27:25,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:27:25,661 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:25,661 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:27:25,661 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:25,661 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:25,661 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:27:25,661 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:25,661 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:27:25,661 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300845661Disabling compacts and flushes for region at 1733300845661Disabling writes for close at 1733300845661Writing region close event to WAL at 1733300845661Closed at 1733300845661 2024-12-04T08:27:25,662 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/.initializing 2024-12-04T08:27:25,662 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/WALs/f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,664 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C36131%2C1733300845541, suffix=, logDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/WALs/f5a5a857f5c5,36131,1733300845541, archiveDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/oldWALs, maxLogs=10 2024-12-04T08:27:25,665 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C36131%2C1733300845541.1733300845665 2024-12-04T08:27:25,669 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/WALs/f5a5a857f5c5,36131,1733300845541/f5a5a857f5c5%2C36131%2C1733300845541.1733300845665 2024-12-04T08:27:25,673 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41683:41683),(127.0.0.1/127.0.0.1:44017:44017)] 2024-12-04T08:27:25,674 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:27:25,674 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:25,674 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,674 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:27:25,676 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:25,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:27:25,677 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:27:25,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:27:25,679 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:27:25,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:27:25,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:27:25,680 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,681 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,681 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,683 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,683 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,683 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:27:25,684 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:27:25,686 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:27:25,686 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=801879, jitterRate=0.019642844796180725}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:27:25,686 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300845674Initializing all the Stores at 1733300845675 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300845675Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300845675Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300845675Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300845675Cleaning up temporary data from old regions at 1733300845683 (+8 ms)Region opened successfully at 1733300845686 (+3 ms) 2024-12-04T08:27:25,687 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:27:25,689 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8d0224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:27:25,690 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:27:25,690 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:27:25,690 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:27:25,690 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:27:25,691 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T08:27:25,691 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T08:27:25,691 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:27:25,693 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:27:25,694 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:27:25,696 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:27:25,697 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:27:25,697 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:27:25,698 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:27:25,698 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:27:25,699 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:27:25,700 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:27:25,701 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:27:25,702 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:27:25,703 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:27:25,705 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:27:25,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:27:25,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:27:25,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,707 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,36131,1733300845541, sessionid=0x1017c95e6660000, setting cluster-up flag (Was=false) 2024-12-04T08:27:25,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,714 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:27:25,715 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:25,723 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:27:25,724 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:25,725 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:27:25,727 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:27:25,727 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:27:25,727 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:27:25,727 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,36131,1733300845541 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:27:25,728 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:27:25,729 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,731 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:27:25,731 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:27:25,732 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,732 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:27:25,732 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300875732 2024-12-04T08:27:25,732 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:27:25,733 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:27:25,734 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:27:25,734 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:27:25,736 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300845734,5,FailOnTimeoutGroup] 2024-12-04T08:27:25,737 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300845736,5,FailOnTimeoutGroup] 2024-12-04T08:27:25,737 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,737 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:27:25,737 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,737 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:27:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:27:25,743 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:27:25,743 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3 2024-12-04T08:27:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:27:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:27:25,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:25,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:27:25,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:27:25,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:25,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:27:25,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:27:25,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:25,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:27:25,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:27:25,756 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:25,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:27:25,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:27:25,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:25,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:25,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:27:25,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740 2024-12-04T08:27:25,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740 2024-12-04T08:27:25,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:27:25,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:27:25,760 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:27:25,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:27:25,762 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:27:25,763 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804502, jitterRate=0.022978246212005615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:27:25,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300845751Initializing all the Stores at 1733300845752 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300845752Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300845752Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300845752Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300845752Cleaning up temporary data from old regions at 1733300845759 (+7 ms)Region opened successfully at 1733300845763 (+4 ms) 2024-12-04T08:27:25,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:27:25,764 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:27:25,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:27:25,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:27:25,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:27:25,764 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:27:25,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300845764Disabling compacts and flushes for region at 1733300845764Disabling writes for close at 1733300845764Writing region close event to WAL at 1733300845764Closed at 1733300845764 2024-12-04T08:27:25,765 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:27:25,765 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:27:25,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:27:25,766 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:27:25,767 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:27:25,802 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(746): ClusterId : dceb7ae4-528d-4a19-a8ca-43c85c63f49c 2024-12-04T08:27:25,802 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:27:25,805 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:27:25,805 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:27:25,807 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:27:25,807 DEBUG [RS:0;f5a5a857f5c5:42763 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67da3b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:27:25,819 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:42763 2024-12-04T08:27:25,819 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:27:25,819 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:27:25,819 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:27:25,819 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,36131,1733300845541 with port=42763, startcode=1733300845588 2024-12-04T08:27:25,820 DEBUG [RS:0;f5a5a857f5c5:42763 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:27:25,822 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47677, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:27:25,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36131 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:25,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36131 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:25,824 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3 2024-12-04T08:27:25,824 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40635 2024-12-04T08:27:25,824 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:27:25,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:27:25,826 DEBUG [RS:0;f5a5a857f5c5:42763 {}] zookeeper.ZKUtil(111): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:25,826 WARN [RS:0;f5a5a857f5c5:42763 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:27:25,826 INFO [RS:0;f5a5a857f5c5:42763 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:27:25,826 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:25,826 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,42763,1733300845588] 2024-12-04T08:27:25,829 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:27:25,831 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:27:25,831 INFO [RS:0;f5a5a857f5c5:42763 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:27:25,831 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,832 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:27:25,832 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:27:25,833 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:27:25,833 DEBUG [RS:0;f5a5a857f5c5:42763 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:27:25,836 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,836 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,836 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,836 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,836 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,836 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42763,1733300845588-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:27:25,850 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:27:25,851 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,42763,1733300845588-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,851 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,851 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.Replication(171): f5a5a857f5c5,42763,1733300845588 started 2024-12-04T08:27:25,864 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:25,864 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,42763,1733300845588, RpcServer on f5a5a857f5c5/172.17.0.2:42763, sessionid=0x1017c95e6660001 2024-12-04T08:27:25,865 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:27:25,865 DEBUG [RS:0;f5a5a857f5c5:42763 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:25,865 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,42763,1733300845588' 2024-12-04T08:27:25,865 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:27:25,865 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,42763,1733300845588' 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:27:25,866 DEBUG [RS:0;f5a5a857f5c5:42763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:27:25,866 INFO [RS:0;f5a5a857f5c5:42763 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:27:25,866 INFO [RS:0;f5a5a857f5c5:42763 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:27:25,917 WARN [f5a5a857f5c5:36131 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:27:25,968 INFO [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C42763%2C1733300845588, suffix=, logDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588, archiveDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/oldWALs, maxLogs=32 2024-12-04T08:27:25,969 INFO [RS:0;f5a5a857f5c5:42763 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C42763%2C1733300845588.1733300845968 2024-12-04T08:27:25,974 INFO [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300845968 2024-12-04T08:27:25,975 DEBUG [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41683:41683),(127.0.0.1/127.0.0.1:44017:44017)] 2024-12-04T08:27:26,168 DEBUG [f5a5a857f5c5:36131 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:27:26,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:26,170 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,42763,1733300845588, state=OPENING 2024-12-04T08:27:26,171 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:27:26,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:26,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:27:26,175 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:27:26,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:27:26,175 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:27:26,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,42763,1733300845588}] 2024-12-04T08:27:26,327 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:27:26,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40277, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:27:26,332 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:27:26,332 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:27:26,334 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C42763%2C1733300845588.meta, suffix=.meta, logDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588, archiveDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/oldWALs, maxLogs=32 2024-12-04T08:27:26,334 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C42763%2C1733300845588.meta.1733300846334.meta 2024-12-04T08:27:26,340 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.meta.1733300846334.meta 2024-12-04T08:27:26,343 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44017:44017),(127.0.0.1/127.0.0.1:41683:41683)] 2024-12-04T08:27:26,344 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:27:26,344 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:27:26,345 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:27:26,345 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:27:26,345 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:27:26,345 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:26,345 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:27:26,345 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:27:26,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:27:26,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:27:26,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:26,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:26,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:27:26,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:27:26,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:26,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:26,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:27:26,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:27:26,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:26,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:26,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:27:26,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:27:26,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:26,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:27:26,350 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:27:26,351 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740 2024-12-04T08:27:26,351 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740 2024-12-04T08:27:26,352 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:27:26,353 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:27:26,353 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:27:26,354 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:27:26,355 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754299, jitterRate=-0.04085974395275116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:27:26,355 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:27:26,355 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300846345Writing region info on filesystem at 1733300846345Initializing all the Stores at 1733300846346 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300846346Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300846346Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300846346Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300846346Cleaning up temporary data from old regions at 1733300846353 (+7 ms)Running coprocessor post-open hooks at 1733300846355 (+2 ms)Region opened successfully at 1733300846355 2024-12-04T08:27:26,356 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300846327 2024-12-04T08:27:26,359 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:27:26,359 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:27:26,360 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:26,361 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,42763,1733300845588, state=OPEN 2024-12-04T08:27:26,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:27:26,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:27:26,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:27:26,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:27:26,366 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:26,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:27:26,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,42763,1733300845588 in 191 msec 2024-12-04T08:27:26,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:27:26,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 604 msec 2024-12-04T08:27:26,371 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:27:26,371 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:27:26,372 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:27:26,373 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,42763,1733300845588, seqNum=-1] 2024-12-04T08:27:26,373 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:27:26,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33763, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:27:26,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 651 msec 2024-12-04T08:27:26,378 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300846378, completionTime=-1 2024-12-04T08:27:26,378 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:27:26,378 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:27:26,380 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:27:26,380 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300906380 2024-12-04T08:27:26,380 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733300966380 2024-12-04T08:27:26,380 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T08:27:26,380 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36131,1733300845541-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,381 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36131,1733300845541-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,381 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36131,1733300845541-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,381 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:36131, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,381 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,381 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,382 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.767sec 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36131,1733300845541-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:27:26,385 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36131,1733300845541-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:27:26,387 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:27:26,387 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:27:26,387 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,36131,1733300845541-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:27:26,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5773e0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:27:26,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,36131,-1 for getting cluster id 2024-12-04T08:27:26,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:27:26,403 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dceb7ae4-528d-4a19-a8ca-43c85c63f49c' 2024-12-04T08:27:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:27:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dceb7ae4-528d-4a19-a8ca-43c85c63f49c" 2024-12-04T08:27:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8850d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:27:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,36131,-1] 2024-12-04T08:27:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:27:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:27:26,405 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:27:26,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab86f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:27:26,406 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:27:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,42763,1733300845588, seqNum=-1] 2024-12-04T08:27:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:27:26,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:27:26,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:26,409 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:27:26,412 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:27:26,412 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T08:27:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f5a5a857f5c5,36131,1733300845541 2024-12-04T08:27:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79a93648 2024-12-04T08:27:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T08:27:26,414 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T08:27:26,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-04T08:27:26,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-04T08:27:26,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:27:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-12-04T08:27:26,417 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T08:27:26,417 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:26,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-12-04T08:27:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:27:26,418 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T08:27:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741835_1011 (size=381) 2024-12-04T08:27:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741835_1011 (size=381) 2024-12-04T08:27:26,426 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6d77e6077b00562e45da0ecd7efac8c, NAME => 'TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3 2024-12-04T08:27:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741836_1012 (size=64) 2024-12-04T08:27:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741836_1012 (size=64) 2024-12-04T08:27:26,432 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:26,432 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing e6d77e6077b00562e45da0ecd7efac8c, disabling compactions & flushes 2024-12-04T08:27:26,432 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,432 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,432 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. after waiting 0 ms 2024-12-04T08:27:26,432 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,432 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,432 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6d77e6077b00562e45da0ecd7efac8c: Waiting for close lock at 1733300846432Disabling compacts and flushes for region at 1733300846432Disabling writes for close at 1733300846432Writing region close event to WAL at 1733300846432Closed at 1733300846432 2024-12-04T08:27:26,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T08:27:26,433 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733300846433"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300846433"}]},"ts":"1733300846433"} 2024-12-04T08:27:26,435 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T08:27:26,436 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T08:27:26,436 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300846436"}]},"ts":"1733300846436"} 2024-12-04T08:27:26,438 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-12-04T08:27:26,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, ASSIGN}] 2024-12-04T08:27:26,439 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, ASSIGN 2024-12-04T08:27:26,440 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, ASSIGN; state=OFFLINE, location=f5a5a857f5c5,42763,1733300845588; forceNewPlan=false, retain=false 2024-12-04T08:27:26,591 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e6d77e6077b00562e45da0ecd7efac8c, regionState=OPENING, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:26,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, ASSIGN because future has completed 2024-12-04T08:27:26,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588}] 2024-12-04T08:27:26,644 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:26,644 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:26,750 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,750 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e6d77e6077b00562e45da0ecd7efac8c, NAME => 'TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:27:26,750 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,750 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:26,750 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,750 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,752 INFO [StoreOpener-e6d77e6077b00562e45da0ecd7efac8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,753 INFO [StoreOpener-e6d77e6077b00562e45da0ecd7efac8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d77e6077b00562e45da0ecd7efac8c columnFamilyName info 2024-12-04T08:27:26,753 DEBUG [StoreOpener-e6d77e6077b00562e45da0ecd7efac8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:26,753 INFO [StoreOpener-e6d77e6077b00562e45da0ecd7efac8c-1 {}] regionserver.HStore(327): Store=e6d77e6077b00562e45da0ecd7efac8c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:27:26,754 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,754 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,754 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,755 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,755 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,756 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,758 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:27:26,758 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e6d77e6077b00562e45da0ecd7efac8c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692466, jitterRate=-0.1194848120212555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:27:26,758 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:26,759 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e6d77e6077b00562e45da0ecd7efac8c: Running coprocessor pre-open hook at 1733300846751Writing region info on filesystem at 1733300846751Initializing all the Stores at 1733300846751Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300846751Cleaning up temporary data from old regions at 1733300846755 (+4 ms)Running coprocessor post-open hooks at 1733300846758 (+3 ms)Region opened successfully at 1733300846759 (+1 ms) 2024-12-04T08:27:26,760 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., pid=6, masterSystemTime=1733300846746 2024-12-04T08:27:26,762 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,762 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:26,763 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e6d77e6077b00562e45da0ecd7efac8c, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:26,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 because future has completed 2024-12-04T08:27:26,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T08:27:26,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 in 172 msec 2024-12-04T08:27:26,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T08:27:26,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, ASSIGN in 330 msec 2024-12-04T08:27:26,772 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T08:27:26,772 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733300846772"}]},"ts":"1733300846772"} 2024-12-04T08:27:26,774 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-12-04T08:27:26,774 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T08:27:26,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 359 msec 2024-12-04T08:27:27,645 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:27,645 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:28,645 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:28,645 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:29,232 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,232 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,232 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,232 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,232 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,232 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,233 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,233 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,250 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,250 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,250 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,251 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,251 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,251 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,256 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,256 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,257 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,646 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:29,646 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:29,764 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:27:29,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,766 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,766 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,782 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,782 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,783 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,783 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,783 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,783 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,786 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,787 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:29,789 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:30,646 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:30,646 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:31,647 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:31,647 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:31,829 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T08:27:31,829 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-12-04T08:27:32,647 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:32,647 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:33,648 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:33,648 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:33,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-04T08:27:33,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-04T08:27:33,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-04T08:27:34,648 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:34,648 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:35,649 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:35,649 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36131 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T08:27:36,440 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-12-04T08:27:36,440 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-12-04T08:27:36,443 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-12-04T08:27:36,443 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2] 2024-12-04T08:27:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:36,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:27:36,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/b0e07f0029ed4c269c342347f594207e is 1080, key is row0001/info:/1733300856446/Put/seqid=0 2024-12-04T08:27:36,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741837_1013 (size=12509) 2024-12-04T08:27:36,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741837_1013 (size=12509) 2024-12-04T08:27:36,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/b0e07f0029ed4c269c342347f594207e 2024-12-04T08:27:36,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/b0e07f0029ed4c269c342347f594207e as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/b0e07f0029ed4c269c342347f594207e 2024-12-04T08:27:36,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/b0e07f0029ed4c269c342347f594207e, entries=7, sequenceid=11, filesize=12.2 K 2024-12-04T08:27:36,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for e6d77e6077b00562e45da0ecd7efac8c in 36ms, sequenceid=11, compaction requested=false 2024-12-04T08:27:36,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:36,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-04T08:27:36,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/51b0a7e7f5f6492c82fd1f7448530663 is 1080, key is row0008/info:/1733300856458/Put/seqid=0 2024-12-04T08:27:36,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741838_1014 (size=29761) 2024-12-04T08:27:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741838_1014 (size=29761) 2024-12-04T08:27:36,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/51b0a7e7f5f6492c82fd1f7448530663 2024-12-04T08:27:36,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/51b0a7e7f5f6492c82fd1f7448530663 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663 2024-12-04T08:27:36,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663, entries=23, sequenceid=37, filesize=29.1 K 2024-12-04T08:27:36,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for e6d77e6077b00562e45da0ecd7efac8c in 20ms, sequenceid=37, compaction requested=false 2024-12-04T08:27:36,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:36,513 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K 2024-12-04T08:27:36,513 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:36,513 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663 because midkey is the same as first or last row 2024-12-04T08:27:36,650 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:36,650 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:37,650 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:37,650 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:38,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:27:38,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/daa8a9eab4e64301901db0eb022dea5a is 1080, key is row0031/info:/1733300856494/Put/seqid=0 2024-12-04T08:27:38,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741839_1015 (size=12509) 2024-12-04T08:27:38,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741839_1015 (size=12509) 2024-12-04T08:27:38,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/daa8a9eab4e64301901db0eb022dea5a 2024-12-04T08:27:38,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/daa8a9eab4e64301901db0eb022dea5a as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/daa8a9eab4e64301901db0eb022dea5a 2024-12-04T08:27:38,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/daa8a9eab4e64301901db0eb022dea5a, entries=7, sequenceid=47, filesize=12.2 K 2024-12-04T08:27:38,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for e6d77e6077b00562e45da0ecd7efac8c in 27ms, sequenceid=47, compaction requested=true 2024-12-04T08:27:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K 2024-12-04T08:27:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663 because midkey is the same as first or last row 2024-12-04T08:27:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d77e6077b00562e45da0ecd7efac8c:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:27:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:38,533 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:27:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:38,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB 2024-12-04T08:27:38,535 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:27:38,535 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): e6d77e6077b00562e45da0ecd7efac8c/info is initiating minor compaction (all files) 2024-12-04T08:27:38,535 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d77e6077b00562e45da0ecd7efac8c/info in TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:38,535 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/b0e07f0029ed4c269c342347f594207e, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/daa8a9eab4e64301901db0eb022dea5a] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp, totalSize=53.5 K 2024-12-04T08:27:38,535 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e07f0029ed4c269c342347f594207e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733300856446 2024-12-04T08:27:38,536 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b0a7e7f5f6492c82fd1f7448530663, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733300856458 2024-12-04T08:27:38,536 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting daa8a9eab4e64301901db0eb022dea5a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733300856494 2024-12-04T08:27:38,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/1c1c03b1995d4b19a7971dd742ae2ace is 1080, key is row0038/info:/1733300858507/Put/seqid=0 2024-12-04T08:27:38,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741840_1016 (size=24376) 2024-12-04T08:27:38,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/1c1c03b1995d4b19a7971dd742ae2ace 2024-12-04T08:27:38,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741840_1016 (size=24376) 2024-12-04T08:27:38,552 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d77e6077b00562e45da0ecd7efac8c#info#compaction#58 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:27:38,553 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/8ffde6ceb2834cbebdb57837e6b20d3c is 1080, key is row0001/info:/1733300856446/Put/seqid=0 2024-12-04T08:27:38,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/1c1c03b1995d4b19a7971dd742ae2ace as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/1c1c03b1995d4b19a7971dd742ae2ace 2024-12-04T08:27:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741841_1017 (size=44978) 2024-12-04T08:27:38,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741841_1017 (size=44978) 2024-12-04T08:27:38,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/1c1c03b1995d4b19a7971dd742ae2ace, entries=18, sequenceid=68, filesize=23.8 K 2024-12-04T08:27:38,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=9.46 KB/9684 for e6d77e6077b00562e45da0ecd7efac8c in 25ms, sequenceid=68, compaction requested=false 2024-12-04T08:27:38,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:38,559 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=77.3 K, sizeToCheck=16.0 K 2024-12-04T08:27:38,559 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:38,559 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663 because midkey is the same as first or last row 2024-12-04T08:27:38,566 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/8ffde6ceb2834cbebdb57837e6b20d3c as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c 2024-12-04T08:27:38,571 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d77e6077b00562e45da0ecd7efac8c/info of e6d77e6077b00562e45da0ecd7efac8c into 8ffde6ceb2834cbebdb57837e6b20d3c(size=43.9 K), total size for store is 67.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:27:38,571 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:38,572 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., storeName=e6d77e6077b00562e45da0ecd7efac8c/info, priority=13, startTime=1733300858533; duration=0sec 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.7 K, sizeToCheck=16.0 K 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c because midkey is the same as first or last row 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.7 K, sizeToCheck=16.0 K 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c because midkey is the same as first or last row 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.7 K, sizeToCheck=16.0 K 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c because midkey is the same as first or last row 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:38,572 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d77e6077b00562e45da0ecd7efac8c:info 2024-12-04T08:27:38,651 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:38,651 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:39,434 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:27:39,435 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,436 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,437 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,437 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,456 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,456 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,456 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,457 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,457 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,457 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,461 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,463 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:39,651 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:39,651 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-12-04T08:27:40,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/e2f3b89cb37245c1b16284dad1e2d126 is 1080, key is row0056/info:/1733300858535/Put/seqid=0 2024-12-04T08:27:40,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741842_1018 (size=15740) 2024-12-04T08:27:40,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741842_1018 (size=15740) 2024-12-04T08:27:40,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/e2f3b89cb37245c1b16284dad1e2d126 2024-12-04T08:27:40,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/e2f3b89cb37245c1b16284dad1e2d126 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/e2f3b89cb37245c1b16284dad1e2d126 2024-12-04T08:27:40,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/e2f3b89cb37245c1b16284dad1e2d126, entries=10, sequenceid=82, filesize=15.4 K 2024-12-04T08:27:40,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=16.81 KB/17216 for e6d77e6077b00562e45da0ecd7efac8c in 28ms, sequenceid=82, compaction requested=true 2024-12-04T08:27:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K 2024-12-04T08:27:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c because midkey is the same as first or last row 2024-12-04T08:27:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d77e6077b00562e45da0ecd7efac8c:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:27:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:40,582 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:27:40,583 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85094 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:27:40,583 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): e6d77e6077b00562e45da0ecd7efac8c/info is initiating minor compaction (all files) 2024-12-04T08:27:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,583 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d77e6077b00562e45da0ecd7efac8c/info in TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:40,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB 2024-12-04T08:27:40,584 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/1c1c03b1995d4b19a7971dd742ae2ace, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/e2f3b89cb37245c1b16284dad1e2d126] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp, totalSize=83.1 K 2024-12-04T08:27:40,584 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ffde6ceb2834cbebdb57837e6b20d3c, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733300856446 2024-12-04T08:27:40,584 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1c03b1995d4b19a7971dd742ae2ace, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733300858507 2024-12-04T08:27:40,585 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f3b89cb37245c1b16284dad1e2d126, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733300858535 2024-12-04T08:27:40,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/00f0f2ee94a3459e83f78179d77ec5cb is 1080, key is row0066/info:/1733300860556/Put/seqid=0 2024-12-04T08:27:40,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741843_1019 (size=24376) 2024-12-04T08:27:40,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741843_1019 (size=24376) 2024-12-04T08:27:40,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/00f0f2ee94a3459e83f78179d77ec5cb 2024-12-04T08:27:40,599 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d77e6077b00562e45da0ecd7efac8c#info#compaction#61 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:27:40,600 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/3458c88d97604486aaea95e9c7df826f is 1080, key is row0001/info:/1733300856446/Put/seqid=0 2024-12-04T08:27:40,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/00f0f2ee94a3459e83f78179d77ec5cb as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/00f0f2ee94a3459e83f78179d77ec5cb 2024-12-04T08:27:40,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741844_1020 (size=75378) 2024-12-04T08:27:40,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741844_1020 (size=75378) 2024-12-04T08:27:40,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T08:27:40,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/00f0f2ee94a3459e83f78179d77ec5cb, entries=18, sequenceid=103, filesize=23.8 K 2024-12-04T08:27:40,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=11.56 KB/11836 for e6d77e6077b00562e45da0ecd7efac8c in 24ms, sequenceid=103, compaction requested=false 2024-12-04T08:27:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=106.9 K, sizeToCheck=16.0 K 2024-12-04T08:27:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:40,607 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c because midkey is the same as first or last row 2024-12-04T08:27:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:55190 deadline: 1733300870605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:40,610 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/3458c88d97604486aaea95e9c7df826f as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f 2024-12-04T08:27:40,615 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6d77e6077b00562e45da0ecd7efac8c/info of e6d77e6077b00562e45da0ecd7efac8c into 3458c88d97604486aaea95e9c7df826f(size=73.6 K), total size for store is 97.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:27:40,615 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d77e6077b00562e45da0ecd7efac8c: 2024-12-04T08:27:40,615 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., storeName=e6d77e6077b00562e45da0ecd7efac8c/info, priority=13, startTime=1733300860582; duration=0sec 2024-12-04T08:27:40,615 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=97.4 K, sizeToCheck=16.0 K 2024-12-04T08:27:40,615 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:40,616 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=97.4 K, sizeToCheck=16.0 K 2024-12-04T08:27:40,616 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:40,616 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=97.4 K, sizeToCheck=16.0 K 2024-12-04T08:27:40,616 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-04T08:27:40,616 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:40,617 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:40,617 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d77e6077b00562e45da0ecd7efac8c:info 2024-12-04T08:27:40,618 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36131 {}] assignment.AssignmentManager(1363): Split request from f5a5a857f5c5,42763,1733300845588, parent={ENCODED => e6d77e6077b00562e45da0ecd7efac8c, NAME => 'TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-12-04T08:27:40,624 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36131 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:40,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T08:27:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T08:27:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2 because the exception is null or not the one we care about 2024-12-04T08:27:40,631 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36131 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e6d77e6077b00562e45da0ecd7efac8c, daughterA=92ff360b6f408ffacaa1f95cb136c4f1, daughterB=070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:40,632 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e6d77e6077b00562e45da0ecd7efac8c, daughterA=92ff360b6f408ffacaa1f95cb136c4f1, daughterB=070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:40,632 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e6d77e6077b00562e45da0ecd7efac8c, daughterA=92ff360b6f408ffacaa1f95cb136c4f1, daughterB=070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:40,632 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e6d77e6077b00562e45da0ecd7efac8c, daughterA=92ff360b6f408ffacaa1f95cb136c4f1, daughterB=070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:40,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, UNASSIGN}] 2024-12-04T08:27:40,640 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, UNASSIGN 2024-12-04T08:27:40,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e6d77e6077b00562e45da0ecd7efac8c, regionState=CLOSING, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:40,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, UNASSIGN because future has completed 2024-12-04T08:27:40,644 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-12-04T08:27:40,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588}] 2024-12-04T08:27:40,652 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:40,652 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:40,802 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,802 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-12-04T08:27:40,803 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing e6d77e6077b00562e45da0ecd7efac8c, disabling compactions & flushes 2024-12-04T08:27:40,803 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:40,803 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:40,803 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. after waiting 0 ms 2024-12-04T08:27:40,803 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:40,803 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing e6d77e6077b00562e45da0ecd7efac8c 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-04T08:27:40,807 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/70017a0ee18749f292599dbb513620f0 is 1080, key is row0084/info:/1733300860584/Put/seqid=0 2024-12-04T08:27:40,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741845_1021 (size=16817) 2024-12-04T08:27:40,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741845_1021 (size=16817) 2024-12-04T08:27:40,814 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/70017a0ee18749f292599dbb513620f0 2024-12-04T08:27:40,820 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/.tmp/info/70017a0ee18749f292599dbb513620f0 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/70017a0ee18749f292599dbb513620f0 2024-12-04T08:27:40,824 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/70017a0ee18749f292599dbb513620f0, entries=11, sequenceid=118, filesize=16.4 K 2024-12-04T08:27:40,826 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for e6d77e6077b00562e45da0ecd7efac8c in 22ms, sequenceid=118, compaction requested=true 2024-12-04T08:27:40,826 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/b0e07f0029ed4c269c342347f594207e, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/daa8a9eab4e64301901db0eb022dea5a, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/1c1c03b1995d4b19a7971dd742ae2ace, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/e2f3b89cb37245c1b16284dad1e2d126] to archive 2024-12-04T08:27:40,827 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T08:27:40,829 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/b0e07f0029ed4c269c342347f594207e to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/b0e07f0029ed4c269c342347f594207e 2024-12-04T08:27:40,830 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/51b0a7e7f5f6492c82fd1f7448530663 2024-12-04T08:27:40,831 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/8ffde6ceb2834cbebdb57837e6b20d3c 2024-12-04T08:27:40,832 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/daa8a9eab4e64301901db0eb022dea5a to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/daa8a9eab4e64301901db0eb022dea5a 2024-12-04T08:27:40,833 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/1c1c03b1995d4b19a7971dd742ae2ace to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/1c1c03b1995d4b19a7971dd742ae2ace 2024-12-04T08:27:40,834 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/e2f3b89cb37245c1b16284dad1e2d126 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/e2f3b89cb37245c1b16284dad1e2d126 2024-12-04T08:27:40,840 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=1 2024-12-04T08:27:40,841 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. 2024-12-04T08:27:40,841 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for e6d77e6077b00562e45da0ecd7efac8c: Waiting for close lock at 1733300860802Running coprocessor pre-close hooks at 1733300860802Disabling compacts and flushes for region at 1733300860802Disabling writes for close at 1733300860803 (+1 ms)Obtaining lock to block concurrent updates at 1733300860803Preparing flush snapshotting stores in e6d77e6077b00562e45da0ecd7efac8c at 1733300860803Finished memstore snapshotting TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., syncing WAL and waiting on mvcc, flushsize=dataSize=11836, getHeapSize=12912, getOffHeapSize=0, getCellsCount=11 at 1733300860803Flushing stores of TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. at 1733300860804 (+1 ms)Flushing e6d77e6077b00562e45da0ecd7efac8c/info: creating writer at 1733300860804Flushing e6d77e6077b00562e45da0ecd7efac8c/info: appending metadata at 1733300860807 (+3 ms)Flushing e6d77e6077b00562e45da0ecd7efac8c/info: closing flushed file at 1733300860807Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66da29d1: reopening flushed file at 1733300860819 (+12 ms)Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for e6d77e6077b00562e45da0ecd7efac8c in 22ms, sequenceid=118, compaction requested=true at 1733300860826 (+7 ms)Writing region close event to WAL at 1733300860837 (+11 ms)Running coprocessor post-close hooks at 1733300860841 (+4 ms)Closed at 1733300860841 2024-12-04T08:27:40,843 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e6d77e6077b00562e45da0ecd7efac8c, regionState=CLOSED 2024-12-04T08:27:40,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 because future has completed 2024-12-04T08:27:40,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T08:27:40,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure e6d77e6077b00562e45da0ecd7efac8c, server=f5a5a857f5c5,42763,1733300845588 in 203 msec 2024-12-04T08:27:40,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T08:27:40,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e6d77e6077b00562e45da0ecd7efac8c, UNASSIGN in 210 msec 2024-12-04T08:27:40,859 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:40,862 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=e6d77e6077b00562e45da0ecd7efac8c, threads=3 2024-12-04T08:27:40,864 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f for region: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,864 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/00f0f2ee94a3459e83f78179d77ec5cb for region: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,864 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/70017a0ee18749f292599dbb513620f0 for region: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,873 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/70017a0ee18749f292599dbb513620f0, top=true 2024-12-04T08:27:40,874 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/00f0f2ee94a3459e83f78179d77ec5cb, top=true 2024-12-04T08:27:40,878 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0 for child: 070a9c13c95f84386e3a31e62b41ce30, parent: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,879 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/70017a0ee18749f292599dbb513620f0 for region: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,881 INFO [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb for child: 070a9c13c95f84386e3a31e62b41ce30, parent: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,881 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/00f0f2ee94a3459e83f78179d77ec5cb for region: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741846_1022 (size=27) 2024-12-04T08:27:40,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741846_1022 (size=27) 2024-12-04T08:27:41,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741847_1023 (size=27) 2024-12-04T08:27:41,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741847_1023 (size=27) 2024-12-04T08:27:41,293 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f for region: e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:27:41,294 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region e6d77e6077b00562e45da0ecd7efac8c Daughter A: [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c] storefiles, Daughter B: [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0] storefiles. 2024-12-04T08:27:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741848_1024 (size=71) 2024-12-04T08:27:41,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741848_1024 (size=71) 2024-12-04T08:27:41,303 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:41,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741849_1025 (size=71) 2024-12-04T08:27:41,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741849_1025 (size=71) 2024-12-04T08:27:41,315 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:41,323 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1 2024-12-04T08:27:41,325 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1 2024-12-04T08:27:41,327 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733300861326"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733300861326"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733300861326"}]},"ts":"1733300861326"} 2024-12-04T08:27:41,327 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733300861326"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300861326"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733300861326"}]},"ts":"1733300861326"} 2024-12-04T08:27:41,327 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733300861326"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733300861326"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733300861326"}]},"ts":"1733300861326"} 2024-12-04T08:27:41,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=92ff360b6f408ffacaa1f95cb136c4f1, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=070a9c13c95f84386e3a31e62b41ce30, ASSIGN}] 2024-12-04T08:27:41,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=92ff360b6f408ffacaa1f95cb136c4f1, ASSIGN 2024-12-04T08:27:41,345 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=070a9c13c95f84386e3a31e62b41ce30, ASSIGN 2024-12-04T08:27:41,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=92ff360b6f408ffacaa1f95cb136c4f1, ASSIGN; state=SPLITTING_NEW, location=f5a5a857f5c5,42763,1733300845588; forceNewPlan=false, retain=false 2024-12-04T08:27:41,345 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=070a9c13c95f84386e3a31e62b41ce30, ASSIGN; state=SPLITTING_NEW, location=f5a5a857f5c5,42763,1733300845588; forceNewPlan=false, retain=false 2024-12-04T08:27:41,496 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=92ff360b6f408ffacaa1f95cb136c4f1, regionState=OPENING, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:41,496 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=070a9c13c95f84386e3a31e62b41ce30, regionState=OPENING, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:41,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=070a9c13c95f84386e3a31e62b41ce30, ASSIGN because future has completed 2024-12-04T08:27:41,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 070a9c13c95f84386e3a31e62b41ce30, server=f5a5a857f5c5,42763,1733300845588}] 2024-12-04T08:27:41,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=92ff360b6f408ffacaa1f95cb136c4f1, ASSIGN because future has completed 2024-12-04T08:27:41,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92ff360b6f408ffacaa1f95cb136c4f1, server=f5a5a857f5c5,42763,1733300845588}] 2024-12-04T08:27:41,653 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:41,653 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:41,655 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:27:41,656 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 92ff360b6f408ffacaa1f95cb136c4f1, NAME => 'TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.', STARTKEY => '', ENDKEY => 'row0062'} 2024-12-04T08:27:41,656 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,656 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:41,656 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,656 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,657 INFO [StoreOpener-92ff360b6f408ffacaa1f95cb136c4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,658 INFO [StoreOpener-92ff360b6f408ffacaa1f95cb136c4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92ff360b6f408ffacaa1f95cb136c4f1 columnFamilyName info 2024-12-04T08:27:41,658 DEBUG [StoreOpener-92ff360b6f408ffacaa1f95cb136c4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:41,668 DEBUG [StoreOpener-92ff360b6f408ffacaa1f95cb136c4f1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c->hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f-bottom 2024-12-04T08:27:41,669 INFO [StoreOpener-92ff360b6f408ffacaa1f95cb136c4f1-1 {}] regionserver.HStore(327): Store=92ff360b6f408ffacaa1f95cb136c4f1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:27:41,669 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,670 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,671 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,671 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,671 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,673 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,673 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 92ff360b6f408ffacaa1f95cb136c4f1; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755489, jitterRate=-0.039346516132354736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:27:41,674 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:27:41,674 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 92ff360b6f408ffacaa1f95cb136c4f1: Running coprocessor pre-open hook at 1733300861656Writing region info on filesystem at 1733300861656Initializing all the Stores at 1733300861657 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300861657Cleaning up temporary data from old regions at 1733300861671 (+14 ms)Running coprocessor post-open hooks at 1733300861674 (+3 ms)Region opened successfully at 1733300861674 2024-12-04T08:27:41,675 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1., pid=13, masterSystemTime=1733300861652 2024-12-04T08:27:41,675 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 92ff360b6f408ffacaa1f95cb136c4f1:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:27:41,675 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:41,675 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T08:27:41,676 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:27:41,676 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 92ff360b6f408ffacaa1f95cb136c4f1/info is initiating minor compaction (all files) 2024-12-04T08:27:41,676 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92ff360b6f408ffacaa1f95cb136c4f1/info in TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:27:41,676 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c->hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f-bottom] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/.tmp, totalSize=73.6 K 2024-12-04T08:27:41,677 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733300856446 2024-12-04T08:27:41,677 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:27:41,677 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:27:41,677 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:27:41,678 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 070a9c13c95f84386e3a31e62b41ce30, NAME => 'TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.', STARTKEY => 'row0062', ENDKEY => ''} 2024-12-04T08:27:41,678 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,678 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:27:41,678 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,678 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,678 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=92ff360b6f408ffacaa1f95cb136c4f1, regionState=OPEN, openSeqNum=122, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:41,679 INFO [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,680 INFO [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 070a9c13c95f84386e3a31e62b41ce30 columnFamilyName info 2024-12-04T08:27:41,680 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-12-04T08:27:41,680 DEBUG [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:27:41,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T08:27:41,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-12-04T08:27:41,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92ff360b6f408ffacaa1f95cb136c4f1, server=f5a5a857f5c5,42763,1733300845588 because future has completed 2024-12-04T08:27:41,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-12-04T08:27:41,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 92ff360b6f408ffacaa1f95cb136c4f1, server=f5a5a857f5c5,42763,1733300845588 in 182 msec 2024-12-04T08:27:41,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=92ff360b6f408ffacaa1f95cb136c4f1, ASSIGN in 341 msec 2024-12-04T08:27:41,693 DEBUG [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c->hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f-top 2024-12-04T08:27:41,696 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92ff360b6f408ffacaa1f95cb136c4f1#info#compaction#63 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:27:41,697 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/.tmp/info/b3e19d77c9fd4072b1cd7925b9dc0b08 is 1080, key is row0001/info:/1733300856446/Put/seqid=0 2024-12-04T08:27:41,697 DEBUG [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb 2024-12-04T08:27:41,701 DEBUG [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0 2024-12-04T08:27:41,701 INFO [StoreOpener-070a9c13c95f84386e3a31e62b41ce30-1 {}] regionserver.HStore(327): Store=070a9c13c95f84386e3a31e62b41ce30/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:27:41,701 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,702 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741850_1026 (size=70862) 2024-12-04T08:27:41,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741850_1026 (size=70862) 2024-12-04T08:27:41,703 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,704 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,704 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,706 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,707 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 070a9c13c95f84386e3a31e62b41ce30; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852529, jitterRate=0.08404707908630371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T08:27:41,707 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:27:41,707 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 070a9c13c95f84386e3a31e62b41ce30: Running coprocessor pre-open hook at 1733300861678Writing region info on filesystem at 1733300861678Initializing all the Stores at 1733300861679 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300861679Cleaning up temporary data from old regions at 1733300861704 (+25 ms)Running coprocessor post-open hooks at 1733300861707 (+3 ms)Region opened successfully at 1733300861707 2024-12-04T08:27:41,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/info/5e7c74d92c7647de8d08b84e273e937d is 193, key is TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30./info:regioninfo/1733300861496/Put/seqid=0 2024-12-04T08:27:41,708 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., pid=12, masterSystemTime=1733300861652 2024-12-04T08:27:41,708 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 2 2024-12-04T08:27:41,708 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:41,708 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:27:41,710 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/.tmp/info/b3e19d77c9fd4072b1cd7925b9dc0b08 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/b3e19d77c9fd4072b1cd7925b9dc0b08 2024-12-04T08:27:41,710 INFO [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:27:41,710 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:27:41,710 INFO [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:27:41,710 DEBUG [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:27:41,710 INFO [RS_OPEN_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:27:41,710 INFO [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c->hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f-top, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=113.8 K 2024-12-04T08:27:41,711 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] compactions.Compactor(225): Compacting 3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733300856446 2024-12-04T08:27:41,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=070a9c13c95f84386e3a31e62b41ce30, regionState=OPEN, openSeqNum=122, regionLocation=f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:41,711 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733300860556 2024-12-04T08:27:41,712 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733300860584 2024-12-04T08:27:41,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 070a9c13c95f84386e3a31e62b41ce30, server=f5a5a857f5c5,42763,1733300845588 because future has completed 2024-12-04T08:27:41,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741851_1027 (size=9847) 2024-12-04T08:27:41,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741851_1027 (size=9847) 2024-12-04T08:27:41,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/info/5e7c74d92c7647de8d08b84e273e937d 2024-12-04T08:27:41,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T08:27:41,717 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 92ff360b6f408ffacaa1f95cb136c4f1/info of 92ff360b6f408ffacaa1f95cb136c4f1 into b3e19d77c9fd4072b1cd7925b9dc0b08(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:27:41,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 070a9c13c95f84386e3a31e62b41ce30, server=f5a5a857f5c5,42763,1733300845588 in 216 msec 2024-12-04T08:27:41,717 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92ff360b6f408ffacaa1f95cb136c4f1: 2024-12-04T08:27:41,717 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1., storeName=92ff360b6f408ffacaa1f95cb136c4f1/info, priority=15, startTime=1733300861675; duration=0sec 2024-12-04T08:27:41,718 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:41,718 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92ff360b6f408ffacaa1f95cb136c4f1:info 2024-12-04T08:27:41,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7 2024-12-04T08:27:41,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=070a9c13c95f84386e3a31e62b41ce30, ASSIGN in 374 msec 2024-12-04T08:27:41,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e6d77e6077b00562e45da0ecd7efac8c, daughterA=92ff360b6f408ffacaa1f95cb136c4f1, daughterB=070a9c13c95f84386e3a31e62b41ce30 in 1.0960 sec 2024-12-04T08:27:41,738 INFO [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#65 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:27:41,739 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/1cad52377a214f77b26e20723ed1b828 is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:27:41,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741852_1028 (size=40830) 2024-12-04T08:27:41,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741852_1028 (size=40830) 2024-12-04T08:27:41,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/ns/62b1cd16c67f43d6b91c0101528beb27 is 43, key is default/ns:d/1733300846374/Put/seqid=0 2024-12-04T08:27:41,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741853_1029 (size=5153) 2024-12-04T08:27:41,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741853_1029 (size=5153) 2024-12-04T08:27:41,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/ns/62b1cd16c67f43d6b91c0101528beb27 2024-12-04T08:27:41,750 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/1cad52377a214f77b26e20723ed1b828 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/1cad52377a214f77b26e20723ed1b828 2024-12-04T08:27:41,755 INFO [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into 1cad52377a214f77b26e20723ed1b828(size=39.9 K), total size for store is 39.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:27:41,755 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:27:41,755 INFO [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300861708; duration=0sec 2024-12-04T08:27:41,756 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:27:41,756 DEBUG [RS:0;f5a5a857f5c5:42763-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:27:41,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/table/f6fbfd2648d94165addc7e381091ef27 is 65, key is TestLogRolling-testLogRolling/table:state/1733300846772/Put/seqid=0 2024-12-04T08:27:41,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741854_1030 (size=5340) 2024-12-04T08:27:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741854_1030 (size=5340) 2024-12-04T08:27:41,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/table/f6fbfd2648d94165addc7e381091ef27 2024-12-04T08:27:41,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/info/5e7c74d92c7647de8d08b84e273e937d as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/info/5e7c74d92c7647de8d08b84e273e937d 2024-12-04T08:27:41,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/info/5e7c74d92c7647de8d08b84e273e937d, entries=30, sequenceid=17, filesize=9.6 K 2024-12-04T08:27:41,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/ns/62b1cd16c67f43d6b91c0101528beb27 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/ns/62b1cd16c67f43d6b91c0101528beb27 2024-12-04T08:27:41,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/ns/62b1cd16c67f43d6b91c0101528beb27, entries=2, sequenceid=17, filesize=5.0 K 2024-12-04T08:27:41,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/table/f6fbfd2648d94165addc7e381091ef27 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/table/f6fbfd2648d94165addc7e381091ef27 2024-12-04T08:27:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/table/f6fbfd2648d94165addc7e381091ef27, entries=2, sequenceid=17, filesize=5.2 K 2024-12-04T08:27:41,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 112ms, sequenceid=17, compaction requested=false 2024-12-04T08:27:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-04T08:27:42,653 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:42,653 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:43,654 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:43,654 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:44,654 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:44,654 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:45,655 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:45,655 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:46,343 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-04T08:27:46,344 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,344 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,345 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,345 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,345 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,345 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,346 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,346 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,366 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,366 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,366 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,367 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,367 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,368 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,373 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,374 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,374 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,378 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-04T08:27:46,655 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:46,655 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:47,656 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:47,656 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:48,657 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:48,657 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:49,657 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:49,657 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:55190 deadline: 1733300880650, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. is not online on f5a5a857f5c5,42763,1733300845588 2024-12-04T08:27:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. is not online on f5a5a857f5c5,42763,1733300845588 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T08:27:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c. is not online on f5a5a857f5c5,42763,1733300845588 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T08:27:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733300846414.e6d77e6077b00562e45da0ecd7efac8c., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=2 from cache 2024-12-04T08:27:50,658 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:50,658 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:51,658 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:51,658 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:52,659 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:52,659 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:53,659 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:53,659 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:54,660 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:54,660 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:55,524 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T08:27:55,661 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:55,661 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:56,661 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:56,661 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:57,662 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:57,662 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:58,662 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:58,662 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:59,663 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:27:59,663 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:00,663 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:00,663 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:01,664 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:01,664 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:02,664 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:02,664 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:03,665 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:03,665 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:04,665 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:04,665 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:05,666 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:05,666 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:06,667 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:06,667 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:06,807 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T08:28:06,807 INFO [master/f5a5a857f5c5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T08:28:07,667 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:07,667 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:08,668 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:08,668 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:09,668 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:09,668 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:10,669 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:10,669 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0095', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., hostname=f5a5a857f5c5,42763,1733300845588, seqNum=122] 2024-12-04T08:28:11,345 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340 2024-12-04T08:28:11,669 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:11,669 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:12,670 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:12,670 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:12,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:28:12,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/970b422b02b34dc3ba71bfb13b77c82d is 1080, key is row0095/info:/1733300890837/Put/seqid=0 2024-12-04T08:28:12,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741855_1031 (size=12513) 2024-12-04T08:28:12,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741855_1031 (size=12513) 2024-12-04T08:28:12,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/970b422b02b34dc3ba71bfb13b77c82d 2024-12-04T08:28:12,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/970b422b02b34dc3ba71bfb13b77c82d as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/970b422b02b34dc3ba71bfb13b77c82d 2024-12-04T08:28:12,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/970b422b02b34dc3ba71bfb13b77c82d, entries=7, sequenceid=132, filesize=12.2 K 2024-12-04T08:28:12,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for 070a9c13c95f84386e3a31e62b41ce30 in 22ms, sequenceid=132, compaction requested=false 2024-12-04T08:28:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:12,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB 2024-12-04T08:28:12,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/bd2d3c2123df4f0f931764fdf0a6bb67 is 1080, key is row0102/info:/1733300892850/Put/seqid=0 2024-12-04T08:28:12,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741856_1032 (size=22238) 2024-12-04T08:28:12,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741856_1032 (size=22238) 2024-12-04T08:28:12,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/bd2d3c2123df4f0f931764fdf0a6bb67 2024-12-04T08:28:12,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/bd2d3c2123df4f0f931764fdf0a6bb67 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/bd2d3c2123df4f0f931764fdf0a6bb67 2024-12-04T08:28:12,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/bd2d3c2123df4f0f931764fdf0a6bb67, entries=16, sequenceid=151, filesize=21.7 K 2024-12-04T08:28:12,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=11.56 KB/11836 for 070a9c13c95f84386e3a31e62b41ce30 in 33ms, sequenceid=151, compaction requested=true 2024-12-04T08:28:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:28:12,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:12,905 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:28:12,906 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 75581 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:28:12,906 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:28:12,906 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:12,906 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/1cad52377a214f77b26e20723ed1b828, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/970b422b02b34dc3ba71bfb13b77c82d, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/bd2d3c2123df4f0f931764fdf0a6bb67] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=73.8 K 2024-12-04T08:28:12,907 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cad52377a214f77b26e20723ed1b828, keycount=33, bloomtype=ROW, size=39.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733300858547 2024-12-04T08:28:12,907 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 970b422b02b34dc3ba71bfb13b77c82d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733300890837 2024-12-04T08:28:12,907 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd2d3c2123df4f0f931764fdf0a6bb67, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733300892850 2024-12-04T08:28:12,917 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#70 average throughput is 57.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:28:12,917 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/155ad0dfaaf94439b1e5fc551ed57055 is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:28:12,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741857_1033 (size=65791) 2024-12-04T08:28:12,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741857_1033 (size=65791) 2024-12-04T08:28:12,927 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/155ad0dfaaf94439b1e5fc551ed57055 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/155ad0dfaaf94439b1e5fc551ed57055 2024-12-04T08:28:12,933 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into 155ad0dfaaf94439b1e5fc551ed57055(size=64.2 K), total size for store is 64.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:28:12,933 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:12,933 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300892905; duration=0sec 2024-12-04T08:28:12,933 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:12,933 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:28:13,670 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:13,670 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:14,671 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:14,671 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:14,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-04T08:28:14,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/c7b8eafa42f54d8eba2c6f3ed9357fbd is 1080, key is row0118/info:/1733300892873/Put/seqid=0 2024-12-04T08:28:14,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741858_1034 (size=17906) 2024-12-04T08:28:14,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741858_1034 (size=17906) 2024-12-04T08:28:14,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/c7b8eafa42f54d8eba2c6f3ed9357fbd 2024-12-04T08:28:14,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/c7b8eafa42f54d8eba2c6f3ed9357fbd as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/c7b8eafa42f54d8eba2c6f3ed9357fbd 2024-12-04T08:28:14,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/c7b8eafa42f54d8eba2c6f3ed9357fbd, entries=12, sequenceid=167, filesize=17.5 K 2024-12-04T08:28:14,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=13.66 KB/13988 for 070a9c13c95f84386e3a31e62b41ce30 in 20ms, sequenceid=167, compaction requested=false 2024-12-04T08:28:14,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:14,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-12-04T08:28:14,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/45b2b2705d3f4a8ea54cbc95deaf93e4 is 1080, key is row0130/info:/1733300894904/Put/seqid=0 2024-12-04T08:28:14,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741859_1035 (size=20078) 2024-12-04T08:28:14,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741859_1035 (size=20078) 2024-12-04T08:28:14,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/45b2b2705d3f4a8ea54cbc95deaf93e4 2024-12-04T08:28:14,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/45b2b2705d3f4a8ea54cbc95deaf93e4 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/45b2b2705d3f4a8ea54cbc95deaf93e4 2024-12-04T08:28:14,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/45b2b2705d3f4a8ea54cbc95deaf93e4, entries=14, sequenceid=184, filesize=19.6 K 2024-12-04T08:28:14,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 070a9c13c95f84386e3a31e62b41ce30 in 21ms, sequenceid=184, compaction requested=true 2024-12-04T08:28:14,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:14,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:28:14,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:14,946 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:28:14,947 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103775 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:28:14,947 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:28:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:14,947 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:14,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T08:28:14,947 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/155ad0dfaaf94439b1e5fc551ed57055, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/c7b8eafa42f54d8eba2c6f3ed9357fbd, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/45b2b2705d3f4a8ea54cbc95deaf93e4] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=101.3 K 2024-12-04T08:28:14,947 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155ad0dfaaf94439b1e5fc551ed57055, keycount=56, bloomtype=ROW, size=64.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733300858547 2024-12-04T08:28:14,948 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b8eafa42f54d8eba2c6f3ed9357fbd, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733300892873 2024-12-04T08:28:14,948 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b2b2705d3f4a8ea54cbc95deaf93e4, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733300894904 2024-12-04T08:28:14,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/3ca8a0109bb84126af97d0ddd97e77a4 is 1080, key is row0144/info:/1733300894925/Put/seqid=0 2024-12-04T08:28:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741860_1036 (size=19000) 2024-12-04T08:28:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741860_1036 (size=19000) 2024-12-04T08:28:14,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/3ca8a0109bb84126af97d0ddd97e77a4 2024-12-04T08:28:14,961 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#74 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:28:14,962 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/ccb7e8b4f2734600bfc5701c10f1482f is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:28:14,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/3ca8a0109bb84126af97d0ddd97e77a4 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3ca8a0109bb84126af97d0ddd97e77a4 2024-12-04T08:28:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741861_1037 (size=93998) 2024-12-04T08:28:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741861_1037 (size=93998) 2024-12-04T08:28:14,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3ca8a0109bb84126af97d0ddd97e77a4, entries=13, sequenceid=200, filesize=18.6 K 2024-12-04T08:28:14,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for 070a9c13c95f84386e3a31e62b41ce30 in 22ms, sequenceid=200, compaction requested=false 2024-12-04T08:28:14,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:14,972 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/ccb7e8b4f2734600bfc5701c10f1482f as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ccb7e8b4f2734600bfc5701c10f1482f 2024-12-04T08:28:14,977 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into ccb7e8b4f2734600bfc5701c10f1482f(size=91.8 K), total size for store is 110.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:28:14,977 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:14,977 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300894945; duration=0sec 2024-12-04T08:28:14,978 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:14,978 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:28:15,671 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:15,671 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:16,672 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:16,672 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:16,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:28:16,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/5033751893f843c6b1989c7c25c5bb11 is 1080, key is row0157/info:/1733300894948/Put/seqid=0 2024-12-04T08:28:16,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741862_1038 (size=12516) 2024-12-04T08:28:16,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741862_1038 (size=12516) 2024-12-04T08:28:16,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/5033751893f843c6b1989c7c25c5bb11 2024-12-04T08:28:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/5033751893f843c6b1989c7c25c5bb11 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/5033751893f843c6b1989c7c25c5bb11 2024-12-04T08:28:16,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/5033751893f843c6b1989c7c25c5bb11, entries=7, sequenceid=211, filesize=12.2 K 2024-12-04T08:28:16,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for 070a9c13c95f84386e3a31e62b41ce30 in 22ms, sequenceid=211, compaction requested=true 2024-12-04T08:28:16,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:16,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:28:16,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:16,982 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:28:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:16,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB 2024-12-04T08:28:16,983 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 125514 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:28:16,983 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:28:16,983 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:16,983 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ccb7e8b4f2734600bfc5701c10f1482f, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3ca8a0109bb84126af97d0ddd97e77a4, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/5033751893f843c6b1989c7c25c5bb11] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=122.6 K 2024-12-04T08:28:16,984 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb7e8b4f2734600bfc5701c10f1482f, keycount=82, bloomtype=ROW, size=91.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733300858547 2024-12-04T08:28:16,984 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca8a0109bb84126af97d0ddd97e77a4, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733300894925 2024-12-04T08:28:16,984 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5033751893f843c6b1989c7c25c5bb11, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733300894948 2024-12-04T08:28:16,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/f54c04fd97f44b7494fa69f21a4642a9 is 1080, key is row0164/info:/1733300896960/Put/seqid=0 2024-12-04T08:28:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741863_1039 (size=22238) 2024-12-04T08:28:16,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741863_1039 (size=22238) 2024-12-04T08:28:16,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/f54c04fd97f44b7494fa69f21a4642a9 2024-12-04T08:28:16,997 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#77 average throughput is 34.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:28:16,997 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/54938d8792354ec895229e33b5da60b6 is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:28:16,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/f54c04fd97f44b7494fa69f21a4642a9 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/f54c04fd97f44b7494fa69f21a4642a9 2024-12-04T08:28:17,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741864_1040 (size=115664) 2024-12-04T08:28:17,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741864_1040 (size=115664) 2024-12-04T08:28:17,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/f54c04fd97f44b7494fa69f21a4642a9, entries=16, sequenceid=230, filesize=21.7 K 2024-12-04T08:28:17,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=12.61 KB/12912 for 070a9c13c95f84386e3a31e62b41ce30 in 22ms, sequenceid=230, compaction requested=false 2024-12-04T08:28:17,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:17,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T08:28:17,007 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/54938d8792354ec895229e33b5da60b6 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/54938d8792354ec895229e33b5da60b6 2024-12-04T08:28:17,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/4a48ee86f35646e8838e125a6197690f is 1080, key is row0180/info:/1733300896983/Put/seqid=0 2024-12-04T08:28:17,013 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into 54938d8792354ec895229e33b5da60b6(size=113.0 K), total size for store is 134.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:28:17,013 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:17,013 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300896982; duration=0sec 2024-12-04T08:28:17,013 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:17,013 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:28:17,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741865_1041 (size=19000) 2024-12-04T08:28:17,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741865_1041 (size=19000) 2024-12-04T08:28:17,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/4a48ee86f35646e8838e125a6197690f 2024-12-04T08:28:17,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/4a48ee86f35646e8838e125a6197690f as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/4a48ee86f35646e8838e125a6197690f 2024-12-04T08:28:17,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/4a48ee86f35646e8838e125a6197690f, entries=13, sequenceid=246, filesize=18.6 K 2024-12-04T08:28:17,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for 070a9c13c95f84386e3a31e62b41ce30 in 21ms, sequenceid=246, compaction requested=true 2024-12-04T08:28:17,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:17,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:28:17,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:17,027 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:28:17,028 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 156902 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:28:17,028 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:28:17,028 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:17,028 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/54938d8792354ec895229e33b5da60b6, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/f54c04fd97f44b7494fa69f21a4642a9, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/4a48ee86f35646e8838e125a6197690f] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=153.2 K 2024-12-04T08:28:17,028 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54938d8792354ec895229e33b5da60b6, keycount=102, bloomtype=ROW, size=113.0 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733300858547 2024-12-04T08:28:17,029 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54c04fd97f44b7494fa69f21a4642a9, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733300896960 2024-12-04T08:28:17,029 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a48ee86f35646e8838e125a6197690f, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733300896983 2024-12-04T08:28:17,040 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#79 average throughput is 67.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:28:17,041 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/653427506ac341c2a6986fc92329bfa3 is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:28:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741866_1042 (size=147233) 2024-12-04T08:28:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741866_1042 (size=147233) 2024-12-04T08:28:17,049 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/653427506ac341c2a6986fc92329bfa3 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/653427506ac341c2a6986fc92329bfa3 2024-12-04T08:28:17,054 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into 653427506ac341c2a6986fc92329bfa3(size=143.8 K), total size for store is 143.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:28:17,054 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:17,054 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300897027; duration=0sec 2024-12-04T08:28:17,054 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:17,054 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:28:17,672 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:17,672 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:18,673 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:18,673 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:19,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-04T08:28:19,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/58358af9574e47189a6d5a19afa728f9 is 1080, key is row0193/info:/1733300899007/Put/seqid=0 2024-12-04T08:28:19,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741867_1043 (size=12518) 2024-12-04T08:28:19,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741867_1043 (size=12518) 2024-12-04T08:28:19,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/58358af9574e47189a6d5a19afa728f9 2024-12-04T08:28:19,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/58358af9574e47189a6d5a19afa728f9 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/58358af9574e47189a6d5a19afa728f9 2024-12-04T08:28:19,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/58358af9574e47189a6d5a19afa728f9, entries=7, sequenceid=258, filesize=12.2 K 2024-12-04T08:28:19,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 070a9c13c95f84386e3a31e62b41ce30 in 22ms, sequenceid=258, compaction requested=false 2024-12-04T08:28:19,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:19,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T08:28:19,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/2b6db746e57d431d9c04907091db71df is 1080, key is row0200/info:/1733300899019/Put/seqid=0 2024-12-04T08:28:19,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741868_1044 (size=19013) 2024-12-04T08:28:19,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741868_1044 (size=19013) 2024-12-04T08:28:19,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/2b6db746e57d431d9c04907091db71df 2024-12-04T08:28:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/2b6db746e57d431d9c04907091db71df as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/2b6db746e57d431d9c04907091db71df 2024-12-04T08:28:19,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/2b6db746e57d431d9c04907091db71df, entries=13, sequenceid=274, filesize=18.6 K 2024-12-04T08:28:19,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for 070a9c13c95f84386e3a31e62b41ce30 in 20ms, sequenceid=274, compaction requested=true 2024-12-04T08:28:19,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:19,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:28:19,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:19,062 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:28:19,063 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 178764 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:28:19,063 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:28:19,063 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:19,063 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/653427506ac341c2a6986fc92329bfa3, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/58358af9574e47189a6d5a19afa728f9, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/2b6db746e57d431d9c04907091db71df] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=174.6 K 2024-12-04T08:28:19,063 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 653427506ac341c2a6986fc92329bfa3, keycount=131, bloomtype=ROW, size=143.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733300858547 2024-12-04T08:28:19,064 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58358af9574e47189a6d5a19afa728f9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733300899007 2024-12-04T08:28:19,064 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6db746e57d431d9c04907091db71df, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733300899019 2024-12-04T08:28:19,074 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#82 average throughput is 77.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:28:19,075 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/3332570aa96b4e5e830be179cb7a20c2 is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:28:19,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741869_1045 (size=168914) 2024-12-04T08:28:19,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741869_1045 (size=168914) 2024-12-04T08:28:19,083 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/3332570aa96b4e5e830be179cb7a20c2 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3332570aa96b4e5e830be179cb7a20c2 2024-12-04T08:28:19,088 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into 3332570aa96b4e5e830be179cb7a20c2(size=165.0 K), total size for store is 165.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:28:19,088 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:19,088 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300899062; duration=0sec 2024-12-04T08:28:19,088 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:19,088 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:28:19,673 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:19,673 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:20,674 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:20,674 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:21,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-04T08:28:21,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/ac059737424c4f62a72a8b4a5ef95be9 is 1080, key is row0213/info:/1733300899042/Put/seqid=0 2024-12-04T08:28:21,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741870_1046 (size=19013) 2024-12-04T08:28:21,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741870_1046 (size=19013) 2024-12-04T08:28:21,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/ac059737424c4f62a72a8b4a5ef95be9 2024-12-04T08:28:21,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/ac059737424c4f62a72a8b4a5ef95be9 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ac059737424c4f62a72a8b4a5ef95be9 2024-12-04T08:28:21,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ac059737424c4f62a72a8b4a5ef95be9, entries=13, sequenceid=291, filesize=18.6 K 2024-12-04T08:28:21,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=13.66 KB/13988 for 070a9c13c95f84386e3a31e62b41ce30 in 20ms, sequenceid=291, compaction requested=false 2024-12-04T08:28:21,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:21,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-12-04T08:28:21,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/0dbc1a6c638b4e178eb75884c19d24ae is 1080, key is row0226/info:/1733300901063/Put/seqid=0 2024-12-04T08:28:21,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741871_1047 (size=20092) 2024-12-04T08:28:21,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741871_1047 (size=20092) 2024-12-04T08:28:21,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/0dbc1a6c638b4e178eb75884c19d24ae 2024-12-04T08:28:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/0dbc1a6c638b4e178eb75884c19d24ae as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/0dbc1a6c638b4e178eb75884c19d24ae 2024-12-04T08:28:21,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/0dbc1a6c638b4e178eb75884c19d24ae, entries=14, sequenceid=308, filesize=19.6 K 2024-12-04T08:28:21,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for 070a9c13c95f84386e3a31e62b41ce30 in 19ms, sequenceid=308, compaction requested=true 2024-12-04T08:28:21,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:21,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 070a9c13c95f84386e3a31e62b41ce30:info, priority=-2147483648, current under compaction store size is 1 2024-12-04T08:28:21,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:21,104 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T08:28:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] regionserver.HRegion(8855): Flush requested on 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:21,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-04T08:28:21,105 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 208019 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T08:28:21,105 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1541): 070a9c13c95f84386e3a31e62b41ce30/info is initiating minor compaction (all files) 2024-12-04T08:28:21,105 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 070a9c13c95f84386e3a31e62b41ce30/info in TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:21,105 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3332570aa96b4e5e830be179cb7a20c2, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ac059737424c4f62a72a8b4a5ef95be9, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/0dbc1a6c638b4e178eb75884c19d24ae] into tmpdir=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp, totalSize=203.1 K 2024-12-04T08:28:21,106 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3332570aa96b4e5e830be179cb7a20c2, keycount=151, bloomtype=ROW, size=165.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733300858547 2024-12-04T08:28:21,107 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac059737424c4f62a72a8b4a5ef95be9, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733300899042 2024-12-04T08:28:21,107 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dbc1a6c638b4e178eb75884c19d24ae, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733300901063 2024-12-04T08:28:21,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/af7351abfde84ee4a45e1b4a3b5665d7 is 1080, key is row0240/info:/1733300901085/Put/seqid=0 2024-12-04T08:28:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741872_1048 (size=17918) 2024-12-04T08:28:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741872_1048 (size=17918) 2024-12-04T08:28:21,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/af7351abfde84ee4a45e1b4a3b5665d7 2024-12-04T08:28:21,124 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 070a9c13c95f84386e3a31e62b41ce30#info#compaction#86 average throughput is 60.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T08:28:21,125 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/a6a09cb07b3e450482750ad0bb5e4c11 is 1080, key is row0062/info:/1733300858547/Put/seqid=0 2024-12-04T08:28:21,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/af7351abfde84ee4a45e1b4a3b5665d7 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/af7351abfde84ee4a45e1b4a3b5665d7 2024-12-04T08:28:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741873_1049 (size=198157) 2024-12-04T08:28:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741873_1049 (size=198157) 2024-12-04T08:28:21,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/af7351abfde84ee4a45e1b4a3b5665d7, entries=12, sequenceid=323, filesize=17.5 K 2024-12-04T08:28:21,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for 070a9c13c95f84386e3a31e62b41ce30 in 26ms, sequenceid=323, compaction requested=false 2024-12-04T08:28:21,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:21,534 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/a6a09cb07b3e450482750ad0bb5e4c11 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/a6a09cb07b3e450482750ad0bb5e4c11 2024-12-04T08:28:21,540 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 070a9c13c95f84386e3a31e62b41ce30/info of 070a9c13c95f84386e3a31e62b41ce30 into a6a09cb07b3e450482750ad0bb5e4c11(size=193.5 K), total size for store is 211.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T08:28:21,540 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:21,540 INFO [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30., storeName=070a9c13c95f84386e3a31e62b41ce30/info, priority=13, startTime=1733300901104; duration=0sec 2024-12-04T08:28:21,540 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T08:28:21,540 DEBUG [RS:0;f5a5a857f5c5:42763-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 070a9c13c95f84386e3a31e62b41ce30:info 2024-12-04T08:28:21,675 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:21,675 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:22,675 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:22,675 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:23,114 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-12-04T08:28:23,115 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C42763%2C1733300845588.1733300903114 2024-12-04T08:28:23,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,121 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300845968 with entries=310, filesize=307.72 KB; new WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300903114 2024-12-04T08:28:23,121 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44017:44017),(127.0.0.1/127.0.0.1:41683:41683)] 2024-12-04T08:28:23,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300845968 is not closed yet, will try archiving it next time 2024-12-04T08:28:23,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741833_1009 (size=315114) 2024-12-04T08:28:23,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741833_1009 (size=315114) 2024-12-04T08:28:23,126 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 92ff360b6f408ffacaa1f95cb136c4f1: 2024-12-04T08:28:23,126 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-12-04T08:28:23,129 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/info/e1437a07e57d44119db8f2c0a856685f is 193, key is TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30./info:regioninfo/1733300861711/Put/seqid=0 2024-12-04T08:28:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741875_1051 (size=6223) 2024-12-04T08:28:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741875_1051 (size=6223) 2024-12-04T08:28:23,134 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/info/e1437a07e57d44119db8f2c0a856685f 2024-12-04T08:28:23,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/.tmp/info/e1437a07e57d44119db8f2c0a856685f as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/info/e1437a07e57d44119db8f2c0a856685f 2024-12-04T08:28:23,143 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/info/e1437a07e57d44119db8f2c0a856685f, entries=5, sequenceid=21, filesize=6.1 K 2024-12-04T08:28:23,144 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false 2024-12-04T08:28:23,144 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-04T08:28:23,144 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 070a9c13c95f84386e3a31e62b41ce30 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB 2024-12-04T08:28:23,147 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/49b85ce792014eacaf32ac34a1ac8c69 is 1080, key is row0252/info:/1733300901105/Put/seqid=0 2024-12-04T08:28:23,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741876_1052 (size=10357) 2024-12-04T08:28:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741876_1052 (size=10357) 2024-12-04T08:28:23,152 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/49b85ce792014eacaf32ac34a1ac8c69 2024-12-04T08:28:23,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/.tmp/info/49b85ce792014eacaf32ac34a1ac8c69 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/49b85ce792014eacaf32ac34a1ac8c69 2024-12-04T08:28:23,160 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/49b85ce792014eacaf32ac34a1ac8c69, entries=5, sequenceid=332, filesize=10.1 K 2024-12-04T08:28:23,162 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 070a9c13c95f84386e3a31e62b41ce30 in 17ms, sequenceid=332, compaction requested=true 2024-12-04T08:28:23,162 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 070a9c13c95f84386e3a31e62b41ce30: 2024-12-04T08:28:23,162 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C42763%2C1733300845588.1733300903162 2024-12-04T08:28:23,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,167 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300903114 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300903162 2024-12-04T08:28:23,167 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41683:41683),(127.0.0.1/127.0.0.1:44017:44017)] 2024-12-04T08:28:23,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300903114 is not closed yet, will try archiving it next time 2024-12-04T08:28:23,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741874_1050 (size=731) 2024-12-04T08:28:23,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741874_1050 (size=731) 2024-12-04T08:28:23,168 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300845968 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/oldWALs/f5a5a857f5c5%2C42763%2C1733300845588.1733300845968 2024-12-04T08:28:23,169 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T08:28:23,169 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/WALs/f5a5a857f5c5,42763,1733300845588/f5a5a857f5c5%2C42763%2C1733300845588.1733300903114 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/oldWALs/f5a5a857f5c5%2C42763%2C1733300845588.1733300903114 2024-12-04T08:28:23,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:28:23,269 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:28:23,269 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:28:23,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:23,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:23,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:28:23,270 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:28:23,270 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1544083512, stopped=false 2024-12-04T08:28:23,270 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,36131,1733300845541 2024-12-04T08:28:23,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:28:23,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:28:23,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:23,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:23,271 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:28:23,272 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:28:23,272 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:28:23,272 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:28:23,272 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:28:23,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:23,272 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,42763,1733300845588' ***** 2024-12-04T08:28:23,273 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:28:23,273 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(3091): Received CLOSE for 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(3091): Received CLOSE for 070a9c13c95f84386e3a31e62b41ce30 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,42763,1733300845588 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:28:23,273 INFO [RS:0;f5a5a857f5c5:42763 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:42763. 2024-12-04T08:28:23,273 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 92ff360b6f408ffacaa1f95cb136c4f1, disabling compactions & flushes 2024-12-04T08:28:23,273 DEBUG [RS:0;f5a5a857f5c5:42763 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:28:23,274 DEBUG [RS:0;f5a5a857f5c5:42763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:23,274 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. after waiting 0 ms 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:28:23,274 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:28:23,274 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:28:23,274 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:28:23,274 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:28:23,274 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-04T08:28:23,274 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1325): Online Regions={92ff360b6f408ffacaa1f95cb136c4f1=TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1., 1588230740=hbase:meta,,1.1588230740, 070a9c13c95f84386e3a31e62b41ce30=TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.} 2024-12-04T08:28:23,274 DEBUG [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1351): Waiting on 070a9c13c95f84386e3a31e62b41ce30, 1588230740, 92ff360b6f408ffacaa1f95cb136c4f1 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:28:23,274 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:28:23,274 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:28:23,274 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c->hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f-bottom] to archive 2024-12-04T08:28:23,275 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T08:28:23,277 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:28:23,277 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f5a5a857f5c5:36131 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T08:28:23,277 WARN [StoreCloser-TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-12-04T08:28:23,279 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-04T08:28:23,279 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:28:23,280 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:28:23,280 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300903274Running coprocessor pre-close hooks at 1733300903274Disabling compacts and flushes for region at 1733300903274Disabling writes for close at 1733300903274Writing region close event to WAL at 1733300903275 (+1 ms)Running coprocessor post-close hooks at 1733300903279 (+4 ms)Closed at 1733300903279 2024-12-04T08:28:23,280 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/92ff360b6f408ffacaa1f95cb136c4f1/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=121 2024-12-04T08:28:23,281 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 92ff360b6f408ffacaa1f95cb136c4f1: Waiting for close lock at 1733300903273Running coprocessor pre-close hooks at 1733300903273Disabling compacts and flushes for region at 1733300903273Disabling writes for close at 1733300903274 (+1 ms)Writing region close event to WAL at 1733300903278 (+4 ms)Running coprocessor post-close hooks at 1733300903281 (+3 ms)Closed at 1733300903281 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733300860624.92ff360b6f408ffacaa1f95cb136c4f1. 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 070a9c13c95f84386e3a31e62b41ce30, disabling compactions & flushes 2024-12-04T08:28:23,281 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. after waiting 0 ms 2024-12-04T08:28:23,281 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:23,282 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c->hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/e6d77e6077b00562e45da0ecd7efac8c/info/3458c88d97604486aaea95e9c7df826f-top, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/1cad52377a214f77b26e20723ed1b828, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/970b422b02b34dc3ba71bfb13b77c82d, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/155ad0dfaaf94439b1e5fc551ed57055, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/bd2d3c2123df4f0f931764fdf0a6bb67, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/c7b8eafa42f54d8eba2c6f3ed9357fbd, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ccb7e8b4f2734600bfc5701c10f1482f, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/45b2b2705d3f4a8ea54cbc95deaf93e4, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3ca8a0109bb84126af97d0ddd97e77a4, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/54938d8792354ec895229e33b5da60b6, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/5033751893f843c6b1989c7c25c5bb11, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/f54c04fd97f44b7494fa69f21a4642a9, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/653427506ac341c2a6986fc92329bfa3, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/4a48ee86f35646e8838e125a6197690f, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/58358af9574e47189a6d5a19afa728f9, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3332570aa96b4e5e830be179cb7a20c2, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/2b6db746e57d431d9c04907091db71df, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ac059737424c4f62a72a8b4a5ef95be9, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/0dbc1a6c638b4e178eb75884c19d24ae] to archive 2024-12-04T08:28:23,283 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T08:28:23,284 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3458c88d97604486aaea95e9c7df826f.e6d77e6077b00562e45da0ecd7efac8c 2024-12-04T08:28:23,285 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-00f0f2ee94a3459e83f78179d77ec5cb 2024-12-04T08:28:23,286 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/1cad52377a214f77b26e20723ed1b828 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/1cad52377a214f77b26e20723ed1b828 2024-12-04T08:28:23,287 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/TestLogRolling-testLogRolling=e6d77e6077b00562e45da0ecd7efac8c-70017a0ee18749f292599dbb513620f0 2024-12-04T08:28:23,288 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/970b422b02b34dc3ba71bfb13b77c82d to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/970b422b02b34dc3ba71bfb13b77c82d 2024-12-04T08:28:23,289 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/155ad0dfaaf94439b1e5fc551ed57055 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/155ad0dfaaf94439b1e5fc551ed57055 2024-12-04T08:28:23,290 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/bd2d3c2123df4f0f931764fdf0a6bb67 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/bd2d3c2123df4f0f931764fdf0a6bb67 2024-12-04T08:28:23,291 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/c7b8eafa42f54d8eba2c6f3ed9357fbd to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/c7b8eafa42f54d8eba2c6f3ed9357fbd 2024-12-04T08:28:23,292 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ccb7e8b4f2734600bfc5701c10f1482f to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ccb7e8b4f2734600bfc5701c10f1482f 2024-12-04T08:28:23,293 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/45b2b2705d3f4a8ea54cbc95deaf93e4 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/45b2b2705d3f4a8ea54cbc95deaf93e4 2024-12-04T08:28:23,294 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3ca8a0109bb84126af97d0ddd97e77a4 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3ca8a0109bb84126af97d0ddd97e77a4 2024-12-04T08:28:23,295 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/54938d8792354ec895229e33b5da60b6 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/54938d8792354ec895229e33b5da60b6 2024-12-04T08:28:23,296 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/5033751893f843c6b1989c7c25c5bb11 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/5033751893f843c6b1989c7c25c5bb11 2024-12-04T08:28:23,297 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/f54c04fd97f44b7494fa69f21a4642a9 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/f54c04fd97f44b7494fa69f21a4642a9 2024-12-04T08:28:23,298 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/653427506ac341c2a6986fc92329bfa3 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/653427506ac341c2a6986fc92329bfa3 2024-12-04T08:28:23,299 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/4a48ee86f35646e8838e125a6197690f to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/4a48ee86f35646e8838e125a6197690f 2024-12-04T08:28:23,300 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/58358af9574e47189a6d5a19afa728f9 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/58358af9574e47189a6d5a19afa728f9 2024-12-04T08:28:23,302 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3332570aa96b4e5e830be179cb7a20c2 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/3332570aa96b4e5e830be179cb7a20c2 2024-12-04T08:28:23,302 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/2b6db746e57d431d9c04907091db71df to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/2b6db746e57d431d9c04907091db71df 2024-12-04T08:28:23,304 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ac059737424c4f62a72a8b4a5ef95be9 to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/ac059737424c4f62a72a8b4a5ef95be9 2024-12-04T08:28:23,305 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/0dbc1a6c638b4e178eb75884c19d24ae to hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/archive/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/info/0dbc1a6c638b4e178eb75884c19d24ae 2024-12-04T08:28:23,305 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1cad52377a214f77b26e20723ed1b828=40830, 970b422b02b34dc3ba71bfb13b77c82d=12513, 155ad0dfaaf94439b1e5fc551ed57055=65791, bd2d3c2123df4f0f931764fdf0a6bb67=22238, c7b8eafa42f54d8eba2c6f3ed9357fbd=17906, ccb7e8b4f2734600bfc5701c10f1482f=93998, 45b2b2705d3f4a8ea54cbc95deaf93e4=20078, 3ca8a0109bb84126af97d0ddd97e77a4=19000, 54938d8792354ec895229e33b5da60b6=115664, 5033751893f843c6b1989c7c25c5bb11=12516, f54c04fd97f44b7494fa69f21a4642a9=22238, 653427506ac341c2a6986fc92329bfa3=147233, 4a48ee86f35646e8838e125a6197690f=19000, 58358af9574e47189a6d5a19afa728f9=12518, 3332570aa96b4e5e830be179cb7a20c2=168914, 2b6db746e57d431d9c04907091db71df=19013, ac059737424c4f62a72a8b4a5ef95be9=19013, 0dbc1a6c638b4e178eb75884c19d24ae=20092] 2024-12-04T08:28:23,308 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/data/default/TestLogRolling-testLogRolling/070a9c13c95f84386e3a31e62b41ce30/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=121 2024-12-04T08:28:23,309 INFO [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:23,309 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 070a9c13c95f84386e3a31e62b41ce30: Waiting for close lock at 1733300903281Running coprocessor pre-close hooks at 1733300903281Disabling compacts and flushes for region at 1733300903281Disabling writes for close at 1733300903281Writing region close event to WAL at 1733300903305 (+24 ms)Running coprocessor post-close hooks at 1733300903309 (+4 ms)Closed at 1733300903309 2024-12-04T08:28:23,309 DEBUG [RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733300860624.070a9c13c95f84386e3a31e62b41ce30. 2024-12-04T08:28:23,474 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,42763,1733300845588; all regions closed. 2024-12-04T08:28:23,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741834_1010 (size=8107) 2024-12-04T08:28:23,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741834_1010 (size=8107) 2024-12-04T08:28:23,479 DEBUG [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/oldWALs 2024-12-04T08:28:23,479 INFO [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C42763%2C1733300845588.meta:.meta(num 1733300846334) 2024-12-04T08:28:23,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741877_1053 (size=778) 2024-12-04T08:28:23,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741877_1053 (size=778) 2024-12-04T08:28:23,483 DEBUG [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/oldWALs 2024-12-04T08:28:23,483 INFO [RS:0;f5a5a857f5c5:42763 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C42763%2C1733300845588:(num 1733300903162) 2024-12-04T08:28:23,483 DEBUG [RS:0;f5a5a857f5c5:42763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:23,483 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:28:23,484 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:28:23,484 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T08:28:23,484 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:28:23,484 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:28:23,484 INFO [RS:0;f5a5a857f5c5:42763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42763 2024-12-04T08:28:23,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,42763,1733300845588 2024-12-04T08:28:23,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:28:23,486 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:28:23,487 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,42763,1733300845588] 2024-12-04T08:28:23,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,42763,1733300845588 already deleted, retry=false 2024-12-04T08:28:23,495 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,42763,1733300845588 expired; onlineServers=0 2024-12-04T08:28:23,495 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,36131,1733300845541' ***** 2024-12-04T08:28:23,495 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:28:23,495 INFO [M:0;f5a5a857f5c5:36131 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:28:23,495 INFO [M:0;f5a5a857f5c5:36131 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:28:23,495 DEBUG [M:0;f5a5a857f5c5:36131 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:28:23,495 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:28:23,495 DEBUG [M:0;f5a5a857f5c5:36131 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:28:23,495 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300845734 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300845734,5,FailOnTimeoutGroup] 2024-12-04T08:28:23,495 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300845736 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300845736,5,FailOnTimeoutGroup] 2024-12-04T08:28:23,495 INFO [M:0;f5a5a857f5c5:36131 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:28:23,496 INFO [M:0;f5a5a857f5c5:36131 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:28:23,496 DEBUG [M:0;f5a5a857f5c5:36131 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:28:23,496 INFO [M:0;f5a5a857f5c5:36131 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:28:23,496 INFO [M:0;f5a5a857f5c5:36131 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:28:23,496 INFO [M:0;f5a5a857f5c5:36131 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:28:23,496 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:28:23,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:28:23,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:23,497 DEBUG [M:0;f5a5a857f5c5:36131 {}] zookeeper.ZKUtil(347): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:28:23,497 WARN [M:0;f5a5a857f5c5:36131 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:28:23,497 INFO [M:0;f5a5a857f5c5:36131 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/.lastflushedseqids 2024-12-04T08:28:23,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741878_1054 (size=228) 2024-12-04T08:28:23,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741878_1054 (size=228) 2024-12-04T08:28:23,503 INFO [M:0;f5a5a857f5c5:36131 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:28:23,503 INFO [M:0;f5a5a857f5c5:36131 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:28:23,504 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:28:23,504 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:23,504 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:23,504 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:28:23,504 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:23,504 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.35 KB 2024-12-04T08:28:23,519 DEBUG [M:0;f5a5a857f5c5:36131 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa2c2f753707490bad609122e0c4a59c is 82, key is hbase:meta,,1/info:regioninfo/1733300846360/Put/seqid=0 2024-12-04T08:28:23,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741879_1055 (size=5672) 2024-12-04T08:28:23,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741879_1055 (size=5672) 2024-12-04T08:28:23,524 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa2c2f753707490bad609122e0c4a59c 2024-12-04T08:28:23,542 DEBUG [M:0;f5a5a857f5c5:36131 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f206ed3629b444df9ccecc6783998df0 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733300846775/Put/seqid=0 2024-12-04T08:28:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741880_1056 (size=7090) 2024-12-04T08:28:23,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741880_1056 (size=7090) 2024-12-04T08:28:23,547 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f206ed3629b444df9ccecc6783998df0 2024-12-04T08:28:23,551 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f206ed3629b444df9ccecc6783998df0 2024-12-04T08:28:23,565 DEBUG [M:0;f5a5a857f5c5:36131 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/703b1358cbd144c09a98ec858d6ae75c is 69, key is f5a5a857f5c5,42763,1733300845588/rs:state/1733300845822/Put/seqid=0 2024-12-04T08:28:23,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741881_1057 (size=5156) 2024-12-04T08:28:23,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741881_1057 (size=5156) 2024-12-04T08:28:23,570 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/703b1358cbd144c09a98ec858d6ae75c 2024-12-04T08:28:23,588 INFO [RS:0;f5a5a857f5c5:42763 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:28:23,588 INFO [RS:0;f5a5a857f5c5:42763 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,42763,1733300845588; zookeeper connection closed. 2024-12-04T08:28:23,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:23,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42763-0x1017c95e6660001, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:23,588 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@116daaa1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@116daaa1 2024-12-04T08:28:23,588 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T08:28:23,588 DEBUG [M:0;f5a5a857f5c5:36131 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c2c2c69ea0d6434e925701858c63fb99 is 52, key is load_balancer_on/state:d/1733300846411/Put/seqid=0 2024-12-04T08:28:23,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741882_1058 (size=5056) 2024-12-04T08:28:23,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741882_1058 (size=5056) 2024-12-04T08:28:23,594 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c2c2c69ea0d6434e925701858c63fb99 2024-12-04T08:28:23,598 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa2c2f753707490bad609122e0c4a59c as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa2c2f753707490bad609122e0c4a59c 2024-12-04T08:28:23,602 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa2c2f753707490bad609122e0c4a59c, entries=8, sequenceid=125, filesize=5.5 K 2024-12-04T08:28:23,603 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f206ed3629b444df9ccecc6783998df0 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f206ed3629b444df9ccecc6783998df0 2024-12-04T08:28:23,607 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f206ed3629b444df9ccecc6783998df0 2024-12-04T08:28:23,607 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f206ed3629b444df9ccecc6783998df0, entries=13, sequenceid=125, filesize=6.9 K 2024-12-04T08:28:23,608 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/703b1358cbd144c09a98ec858d6ae75c as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/703b1358cbd144c09a98ec858d6ae75c 2024-12-04T08:28:23,611 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/703b1358cbd144c09a98ec858d6ae75c, entries=1, sequenceid=125, filesize=5.0 K 2024-12-04T08:28:23,612 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c2c2c69ea0d6434e925701858c63fb99 as hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c2c2c69ea0d6434e925701858c63fb99 2024-12-04T08:28:23,616 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40635/user/jenkins/test-data/7c926e58-0b42-4292-0533-756fe7fe51a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c2c2c69ea0d6434e925701858c63fb99, entries=1, sequenceid=125, filesize=4.9 K 2024-12-04T08:28:23,617 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=125, compaction requested=false 2024-12-04T08:28:23,619 INFO [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:23,619 DEBUG [M:0;f5a5a857f5c5:36131 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300903503Disabling compacts and flushes for region at 1733300903503Disabling writes for close at 1733300903504 (+1 ms)Obtaining lock to block concurrent updates at 1733300903504Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300903504Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64808, getOffHeapSize=0, getCellsCount=148 at 1733300903504Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300903505 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300903505Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300903519 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300903519Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300903528 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300903542 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300903542Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300903551 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300903565 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300903565Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300903574 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300903588 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300903588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475de601: reopening flushed file at 1733300903598 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b01eb11: reopening flushed file at 1733300903603 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25519234: reopening flushed file at 1733300903607 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cfd7587: reopening flushed file at 1733300903611 (+4 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=125, compaction requested=false at 1733300903617 (+6 ms)Writing region close event to WAL at 1733300903619 (+2 ms)Closed at 1733300903619 2024-12-04T08:28:23,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35299 is added to blk_1073741830_1006 (size=61320) 2024-12-04T08:28:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44871 is added to blk_1073741830_1006 (size=61320) 2024-12-04T08:28:23,623 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:28:23,623 INFO [M:0;f5a5a857f5c5:36131 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:28:23,623 INFO [M:0;f5a5a857f5c5:36131 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36131 2024-12-04T08:28:23,623 INFO [M:0;f5a5a857f5c5:36131 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:28:23,676 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:23,676 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:23,725 INFO [M:0;f5a5a857f5c5:36131 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:28:23,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:23,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36131-0x1017c95e6660000, quorum=127.0.0.1:60685, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:23,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d7cc900{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:28:23,727 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20d96a0d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:28:23,727 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:28:23,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@425d5d71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:28:23,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@314e7370{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir/,STOPPED} 2024-12-04T08:28:23,729 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:28:23,729 WARN [BP-1748760397-172.17.0.2-1733300844727 heartbeating to localhost/127.0.0.1:40635 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:28:23,730 WARN [BP-1748760397-172.17.0.2-1733300844727 heartbeating to localhost/127.0.0.1:40635 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1748760397-172.17.0.2-1733300844727 (Datanode Uuid 87c9e3e8-a639-4268-a848-52e83c69b4bd) service to localhost/127.0.0.1:40635 2024-12-04T08:28:23,730 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:28:23,730 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data3/current/BP-1748760397-172.17.0.2-1733300844727 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:23,730 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data4/current/BP-1748760397-172.17.0.2-1733300844727 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:23,730 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:28:23,732 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5411f427{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:28:23,733 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10e59f6e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:28:23,733 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:28:23,733 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50f209dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:28:23,733 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d639fc0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir/,STOPPED} 2024-12-04T08:28:23,734 WARN [BP-1748760397-172.17.0.2-1733300844727 heartbeating to localhost/127.0.0.1:40635 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:28:23,734 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:28:23,734 WARN [BP-1748760397-172.17.0.2-1733300844727 heartbeating to localhost/127.0.0.1:40635 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1748760397-172.17.0.2-1733300844727 (Datanode Uuid 34e809a6-b925-43e6-ae10-ee8f2ada7b91) service to localhost/127.0.0.1:40635 2024-12-04T08:28:23,734 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:28:23,735 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data1/current/BP-1748760397-172.17.0.2-1733300844727 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:23,735 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/cluster_31bff1b8-0376-c037-a9b6-43b3cb6d3738/data/data2/current/BP-1748760397-172.17.0.2-1733300844727 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:23,735 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:28:23,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25570184{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:28:23,741 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@148b0695{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:28:23,741 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:28:23,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@314bb674{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:28:23,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@240fc28c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir/,STOPPED} 2024-12-04T08:28:23,748 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:28:23,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:28:23,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=232 (was 207) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40635 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40635 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40635 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40635 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:40635 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40635 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40635 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40635 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=518 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=6 (was 16), ProcessCount=11 (was 11), AvailableMemoryMB=6353 (was 6406) 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=232, OpenFileDescriptor=518, MaxFileDescriptor=1048576, SystemLoadAverage=6, ProcessCount=11, AvailableMemoryMB=6353 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.log.dir so I do NOT create it in target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/966c0658-617b-bc8a-1f59-b684b5e816d1/hadoop.tmp.dir so I do NOT create it in target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9, deleteOnExit=true 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/test.cache.data in system properties and HBase conf 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir in system properties and HBase conf 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T08:28:23,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T08:28:23,792 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/nfs.dump.dir in system properties and HBase conf 2024-12-04T08:28:23,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/java.io.tmpdir in system properties and HBase conf 2024-12-04T08:28:23,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T08:28:23,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T08:28:23,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T08:28:23,805 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:28:23,838 INFO [regionserver/f5a5a857f5c5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:28:23,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:28:23,873 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:28:23,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:28:23,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:28:23,875 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:28:23,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:28:23,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@512e80eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:28:23,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@eedc0c0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:28:23,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:28:23,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T08:28:23,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T08:28:23,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-04T08:28:23,990 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6dda9ea2{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/java.io.tmpdir/jetty-localhost-36945-hadoop-hdfs-3_4_1-tests_jar-_-any-1033294080956541086/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:28:23,990 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a96c77e{HTTP/1.1, (http/1.1)}{localhost:36945} 2024-12-04T08:28:23,990 INFO [Time-limited test {}] server.Server(415): Started @295404ms 2024-12-04T08:28:24,003 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-04T08:28:24,057 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:28:24,060 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:28:24,061 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:28:24,061 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:28:24,061 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:28:24,061 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e32ebb8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:28:24,061 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76ccace4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:28:24,174 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@333ec6ee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/java.io.tmpdir/jetty-localhost-35469-hadoop-hdfs-3_4_1-tests_jar-_-any-11072366510008328431/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:28:24,174 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47f122ec{HTTP/1.1, (http/1.1)}{localhost:35469} 2024-12-04T08:28:24,175 INFO [Time-limited test {}] server.Server(415): Started @295588ms 2024-12-04T08:28:24,176 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:28:24,204 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T08:28:24,207 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T08:28:24,207 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T08:28:24,207 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T08:28:24,207 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T08:28:24,208 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7748f5df{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir/,AVAILABLE} 2024-12-04T08:28:24,208 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@637d6eb9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T08:28:24,270 WARN [Thread-2472 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data1/current/BP-1517250604-172.17.0.2-1733300903811/current, will proceed with Du for space computation calculation, 2024-12-04T08:28:24,270 WARN [Thread-2473 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data2/current/BP-1517250604-172.17.0.2-1733300903811/current, will proceed with Du for space computation calculation, 2024-12-04T08:28:24,292 WARN [Thread-2451 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:28:24,294 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2628a76320264cea with lease ID 0x47efd49c81f9d79b: Processing first storage report for DS-7452a79c-d976-44e0-9cfa-aa2081b4a41e from datanode DatanodeRegistration(127.0.0.1:46393, datanodeUuid=0d50fccb-8ba0-40ca-a3a3-067aa210ee97, infoPort=45527, infoSecurePort=0, ipcPort=44025, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811) 2024-12-04T08:28:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2628a76320264cea with lease ID 0x47efd49c81f9d79b: from storage DS-7452a79c-d976-44e0-9cfa-aa2081b4a41e node DatanodeRegistration(127.0.0.1:46393, datanodeUuid=0d50fccb-8ba0-40ca-a3a3-067aa210ee97, infoPort=45527, infoSecurePort=0, ipcPort=44025, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:28:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2628a76320264cea with lease ID 0x47efd49c81f9d79b: Processing first storage report for DS-a264011d-b5d1-4575-ba95-9d063cdbfc07 from datanode DatanodeRegistration(127.0.0.1:46393, datanodeUuid=0d50fccb-8ba0-40ca-a3a3-067aa210ee97, infoPort=45527, infoSecurePort=0, ipcPort=44025, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811) 2024-12-04T08:28:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2628a76320264cea with lease ID 0x47efd49c81f9d79b: from storage DS-a264011d-b5d1-4575-ba95-9d063cdbfc07 node DatanodeRegistration(127.0.0.1:46393, datanodeUuid=0d50fccb-8ba0-40ca-a3a3-067aa210ee97, infoPort=45527, infoSecurePort=0, ipcPort=44025, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:28:24,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f9a8217{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/java.io.tmpdir/jetty-localhost-41833-hadoop-hdfs-3_4_1-tests_jar-_-any-1327555549746209320/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:28:24,332 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@35c58925{HTTP/1.1, (http/1.1)}{localhost:41833} 2024-12-04T08:28:24,332 INFO [Time-limited test {}] server.Server(415): Started @295746ms 2024-12-04T08:28:24,334 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T08:28:24,441 WARN [Thread-2498 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data3/current/BP-1517250604-172.17.0.2-1733300903811/current, will proceed with Du for space computation calculation, 2024-12-04T08:28:24,441 WARN [Thread-2499 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data4/current/BP-1517250604-172.17.0.2-1733300903811/current, will proceed with Du for space computation calculation, 2024-12-04T08:28:24,456 WARN [Thread-2487 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T08:28:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7e02ee995a49691 with lease ID 0x47efd49c81f9d79c: Processing first storage report for DS-1a664fbf-a8e1-493d-bb89-8bf51f4ac427 from datanode DatanodeRegistration(127.0.0.1:43795, datanodeUuid=6a263e7a-01ae-4433-8401-f40e78851c33, infoPort=40001, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811) 2024-12-04T08:28:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7e02ee995a49691 with lease ID 0x47efd49c81f9d79c: from storage DS-1a664fbf-a8e1-493d-bb89-8bf51f4ac427 node DatanodeRegistration(127.0.0.1:43795, datanodeUuid=6a263e7a-01ae-4433-8401-f40e78851c33, infoPort=40001, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:28:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7e02ee995a49691 with lease ID 0x47efd49c81f9d79c: Processing first storage report for DS-18facc8c-130c-4bc9-85a8-7d22342acd15 from datanode DatanodeRegistration(127.0.0.1:43795, datanodeUuid=6a263e7a-01ae-4433-8401-f40e78851c33, infoPort=40001, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811) 2024-12-04T08:28:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7e02ee995a49691 with lease ID 0x47efd49c81f9d79c: from storage DS-18facc8c-130c-4bc9-85a8-7d22342acd15 node DatanodeRegistration(127.0.0.1:43795, datanodeUuid=6a263e7a-01ae-4433-8401-f40e78851c33, infoPort=40001, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1966604011;c=1733300903811), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T08:28:24,557 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43 2024-12-04T08:28:24,559 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/zookeeper_0, clientPort=62959, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T08:28:24,560 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62959 2024-12-04T08:28:24,560 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:28:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741825_1001 (size=7) 2024-12-04T08:28:24,570 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5 with version=8 2024-12-04T08:28:24,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39713/user/jenkins/test-data/2528f52f-9a25-0f8e-ae1a-fa2d021fefce/hbase-staging 2024-12-04T08:28:24,572 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T08:28:24,572 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:28:24,573 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33343 2024-12-04T08:28:24,574 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33343 connecting to ZooKeeper ensemble=127.0.0.1:62959 2024-12-04T08:28:24,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333430x0, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:28:24,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33343-0x1017c96ccfe0000 connected 2024-12-04T08:28:24,602 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:28:24,605 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5, hbase.cluster.distributed=false 2024-12-04T08:28:24,607 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:28:24,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33343 2024-12-04T08:28:24,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33343 2024-12-04T08:28:24,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33343 2024-12-04T08:28:24,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33343 2024-12-04T08:28:24,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33343 2024-12-04T08:28:24,623 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f5a5a857f5c5:0 server-side Connection retries=45 2024-12-04T08:28:24,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T08:28:24,624 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46109 2024-12-04T08:28:24,625 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46109 connecting to ZooKeeper ensemble=127.0.0.1:62959 2024-12-04T08:28:24,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461090x0, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T08:28:24,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461090x0, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:28:24,632 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46109-0x1017c96ccfe0001 connected 2024-12-04T08:28:24,632 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T08:28:24,633 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T08:28:24,633 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T08:28:24,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T08:28:24,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46109 2024-12-04T08:28:24,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46109 2024-12-04T08:28:24,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46109 2024-12-04T08:28:24,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46109 2024-12-04T08:28:24,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46109 2024-12-04T08:28:24,647 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f5a5a857f5c5:33343 2024-12-04T08:28:24,648 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:28:24,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:28:24,650 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T08:28:24,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,652 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T08:28:24,652 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f5a5a857f5c5,33343,1733300904572 from backup master directory 2024-12-04T08:28:24,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:28:24,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T08:28:24,655 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:28:24,655 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,659 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/hbase.id] with ID: 87cabcae-6028-4b29-b095-5793c437b08b 2024-12-04T08:28:24,659 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/.tmp/hbase.id 2024-12-04T08:28:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:28:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741826_1002 (size=42) 2024-12-04T08:28:24,666 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/.tmp/hbase.id]:[hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/hbase.id] 2024-12-04T08:28:24,676 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:24,676 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T08:28:24,676 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:24,676 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:24,678 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-04T08:28:24,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:28:24,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741827_1003 (size=196) 2024-12-04T08:28:24,687 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T08:28:24,688 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T08:28:24,688 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:28:24,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:28:24,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741828_1004 (size=1189) 2024-12-04T08:28:24,695 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store 2024-12-04T08:28:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:28:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741829_1005 (size=34) 2024-12-04T08:28:24,700 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:28:24,701 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:28:24,701 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:24,701 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:24,701 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:28:24,701 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:24,701 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:24,701 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300904701Disabling compacts and flushes for region at 1733300904701Disabling writes for close at 1733300904701Writing region close event to WAL at 1733300904701Closed at 1733300904701 2024-12-04T08:28:24,702 WARN [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/.initializing 2024-12-04T08:28:24,702 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/WALs/f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,704 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C33343%2C1733300904572, suffix=, logDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/WALs/f5a5a857f5c5,33343,1733300904572, archiveDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/oldWALs, maxLogs=10 2024-12-04T08:28:24,704 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C33343%2C1733300904572.1733300904704 2024-12-04T08:28:24,708 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/WALs/f5a5a857f5c5,33343,1733300904572/f5a5a857f5c5%2C33343%2C1733300904572.1733300904704 2024-12-04T08:28:24,713 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40001:40001),(127.0.0.1/127.0.0.1:45527:45527)] 2024-12-04T08:28:24,713 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:28:24,713 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:28:24,713 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,714 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T08:28:24,716 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:24,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T08:28:24,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:28:24,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T08:28:24,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:28:24,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T08:28:24,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T08:28:24,720 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,721 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,721 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,722 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,722 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,723 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T08:28:24,724 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T08:28:24,725 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:28:24,726 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692298, jitterRate=-0.11969813704490662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T08:28:24,727 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733300904714Initializing all the Stores at 1733300904714Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300904714Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300904714Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300904714Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300904714Cleaning up temporary data from old regions at 1733300904722 (+8 ms)Region opened successfully at 1733300904727 (+5 ms) 2024-12-04T08:28:24,727 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T08:28:24,730 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8c158e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:28:24,731 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T08:28:24,731 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T08:28:24,731 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T08:28:24,731 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T08:28:24,731 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T08:28:24,732 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T08:28:24,732 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T08:28:24,734 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T08:28:24,735 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T08:28:24,736 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T08:28:24,737 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T08:28:24,737 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T08:28:24,739 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T08:28:24,739 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T08:28:24,740 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T08:28:24,742 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T08:28:24,742 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T08:28:24,744 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T08:28:24,745 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T08:28:24,746 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T08:28:24,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:28:24,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T08:28:24,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,749 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f5a5a857f5c5,33343,1733300904572, sessionid=0x1017c96ccfe0000, setting cluster-up flag (Was=false) 2024-12-04T08:28:24,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,759 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T08:28:24,760 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:24,767 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T08:28:24,768 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:24,769 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T08:28:24,771 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T08:28:24,771 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T08:28:24,771 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T08:28:24,771 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f5a5a857f5c5,33343,1733300904572 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=5, maxPoolSize=5 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f5a5a857f5c5:0, corePoolSize=10, maxPoolSize=10 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:28:24,772 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733300934773 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T08:28:24,773 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,774 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T08:28:24,774 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T08:28:24,774 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300904774,5,FailOnTimeoutGroup] 2024-12-04T08:28:24,774 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300904774,5,FailOnTimeoutGroup] 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,774 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,775 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,775 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T08:28:24,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:28:24,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741831_1007 (size=1321) 2024-12-04T08:28:24,782 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T08:28:24,782 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5 2024-12-04T08:28:24,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:28:24,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741832_1008 (size=32) 2024-12-04T08:28:24,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:28:24,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:28:24,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:28:24,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:24,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:28:24,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:28:24,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:24,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:28:24,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:28:24,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:24,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:28:24,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:28:24,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:24,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:24,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:28:24,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740 2024-12-04T08:28:24,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740 2024-12-04T08:28:24,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:28:24,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:28:24,797 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:28:24,797 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:28:24,799 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T08:28:24,799 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873480, jitterRate=0.11068817973136902}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:28:24,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733300904788Initializing all the Stores at 1733300904789 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300904789Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300904789Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300904789Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300904789Cleaning up temporary data from old regions at 1733300904796 (+7 ms)Region opened successfully at 1733300904800 (+4 ms) 2024-12-04T08:28:24,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:28:24,800 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:28:24,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:28:24,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:28:24,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:28:24,800 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:28:24,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300904800Disabling compacts and flushes for region at 1733300904800Disabling writes for close at 1733300904800Writing region close event to WAL at 1733300904800Closed at 1733300904800 2024-12-04T08:28:24,801 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:28:24,801 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T08:28:24,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T08:28:24,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:28:24,803 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T08:28:24,837 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(746): ClusterId : 87cabcae-6028-4b29-b095-5793c437b08b 2024-12-04T08:28:24,837 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T08:28:24,839 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T08:28:24,839 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T08:28:24,841 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T08:28:24,841 DEBUG [RS:0;f5a5a857f5c5:46109 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661ee605, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f5a5a857f5c5/172.17.0.2:0 2024-12-04T08:28:24,853 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f5a5a857f5c5:46109 2024-12-04T08:28:24,853 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T08:28:24,853 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T08:28:24,853 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T08:28:24,854 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(2659): reportForDuty to master=f5a5a857f5c5,33343,1733300904572 with port=46109, startcode=1733300904623 2024-12-04T08:28:24,854 DEBUG [RS:0;f5a5a857f5c5:46109 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T08:28:24,856 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34877, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T08:28:24,857 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33343 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:24,857 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33343 {}] master.ServerManager(517): Registering regionserver=f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:24,858 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5 2024-12-04T08:28:24,858 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34681 2024-12-04T08:28:24,858 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T08:28:24,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:28:24,861 DEBUG [RS:0;f5a5a857f5c5:46109 {}] zookeeper.ZKUtil(111): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:24,861 WARN [RS:0;f5a5a857f5c5:46109 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T08:28:24,861 INFO [RS:0;f5a5a857f5c5:46109 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:28:24,861 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:24,862 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f5a5a857f5c5,46109,1733300904623] 2024-12-04T08:28:24,864 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T08:28:24,865 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T08:28:24,866 INFO [RS:0;f5a5a857f5c5:46109 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T08:28:24,866 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,866 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T08:28:24,867 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T08:28:24,867 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=2, maxPoolSize=2 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f5a5a857f5c5:0, corePoolSize=1, maxPoolSize=1 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:28:24,867 DEBUG [RS:0;f5a5a857f5c5:46109 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f5a5a857f5c5:0, corePoolSize=3, maxPoolSize=3 2024-12-04T08:28:24,868 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,868 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,868 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,868 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,868 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,868 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,46109,1733300904623-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:28:24,882 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T08:28:24,882 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,46109,1733300904623-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,882 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,882 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.Replication(171): f5a5a857f5c5,46109,1733300904623 started 2024-12-04T08:28:24,895 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:24,895 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1482): Serving as f5a5a857f5c5,46109,1733300904623, RpcServer on f5a5a857f5c5/172.17.0.2:46109, sessionid=0x1017c96ccfe0001 2024-12-04T08:28:24,895 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T08:28:24,895 DEBUG [RS:0;f5a5a857f5c5:46109 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:24,895 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,46109,1733300904623' 2024-12-04T08:28:24,895 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f5a5a857f5c5,46109,1733300904623' 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T08:28:24,896 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T08:28:24,897 DEBUG [RS:0;f5a5a857f5c5:46109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T08:28:24,897 INFO [RS:0;f5a5a857f5c5:46109 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T08:28:24,897 INFO [RS:0;f5a5a857f5c5:46109 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T08:28:24,953 WARN [f5a5a857f5c5:33343 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T08:28:24,998 INFO [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C46109%2C1733300904623, suffix=, logDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/f5a5a857f5c5,46109,1733300904623, archiveDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs, maxLogs=32 2024-12-04T08:28:24,999 INFO [RS:0;f5a5a857f5c5:46109 {}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C46109%2C1733300904623.1733300904999 2024-12-04T08:28:25,004 INFO [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/f5a5a857f5c5,46109,1733300904623/f5a5a857f5c5%2C46109%2C1733300904623.1733300904999 2024-12-04T08:28:25,005 DEBUG [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40001:40001),(127.0.0.1/127.0.0.1:45527:45527)] 2024-12-04T08:28:25,203 DEBUG [f5a5a857f5c5:33343 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T08:28:25,204 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:25,205 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,46109,1733300904623, state=OPENING 2024-12-04T08:28:25,207 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T08:28:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:25,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:25,208 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T08:28:25,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:28:25,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:28:25,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,46109,1733300904623}] 2024-12-04T08:28:25,361 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T08:28:25,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41131, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T08:28:25,366 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T08:28:25,366 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:28:25,367 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f5a5a857f5c5%2C46109%2C1733300904623.meta, suffix=.meta, logDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/f5a5a857f5c5,46109,1733300904623, archiveDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs, maxLogs=32 2024-12-04T08:28:25,368 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor f5a5a857f5c5%2C46109%2C1733300904623.meta.1733300905368.meta 2024-12-04T08:28:25,373 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/f5a5a857f5c5,46109,1733300904623/f5a5a857f5c5%2C46109%2C1733300904623.meta.1733300905368.meta 2024-12-04T08:28:25,377 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40001:40001),(127.0.0.1/127.0.0.1:45527:45527)] 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T08:28:25,381 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T08:28:25,381 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T08:28:25,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T08:28:25,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T08:28:25,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:25,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:25,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T08:28:25,388 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T08:28:25,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:25,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:25,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T08:28:25,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T08:28:25,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:25,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:25,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T08:28:25,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T08:28:25,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T08:28:25,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T08:28:25,390 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T08:28:25,391 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740 2024-12-04T08:28:25,391 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740 2024-12-04T08:28:25,392 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T08:28:25,392 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T08:28:25,393 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T08:28:25,394 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T08:28:25,394 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755379, jitterRate=-0.03948655724525452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T08:28:25,394 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T08:28:25,395 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733300905382Writing region info on filesystem at 1733300905382Initializing all the Stores at 1733300905382Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300905382Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300905386 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733300905386Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733300905386Cleaning up temporary data from old regions at 1733300905392 (+6 ms)Running coprocessor post-open hooks at 1733300905394 (+2 ms)Region opened successfully at 1733300905395 (+1 ms) 2024-12-04T08:28:25,396 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733300905361 2024-12-04T08:28:25,398 DEBUG [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T08:28:25,398 INFO [RS_OPEN_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T08:28:25,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:25,400 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f5a5a857f5c5,46109,1733300904623, state=OPEN 2024-12-04T08:28:25,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:28:25,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T08:28:25,407 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:25,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:28:25,407 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T08:28:25,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T08:28:25,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f5a5a857f5c5,46109,1733300904623 in 199 msec 2024-12-04T08:28:25,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T08:28:25,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-12-04T08:28:25,412 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T08:28:25,412 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T08:28:25,413 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:28:25,413 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,46109,1733300904623, seqNum=-1] 2024-12-04T08:28:25,414 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:28:25,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41769, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:28:25,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 648 msec 2024-12-04T08:28:25,419 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733300905419, completionTime=-1 2024-12-04T08:28:25,420 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T08:28:25,420 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T08:28:25,421 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T08:28:25,421 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733300965421 2024-12-04T08:28:25,421 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733301025421 2024-12-04T08:28:25,421 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-04T08:28:25,422 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,33343,1733300904572-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,422 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,33343,1733300904572-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,422 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,33343,1733300904572-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,422 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f5a5a857f5c5:33343, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,422 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,422 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,423 DEBUG [master/f5a5a857f5c5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.771sec 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,33343,1733300904572-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T08:28:25,426 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,33343,1733300904572-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T08:28:25,428 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T08:28:25,428 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T08:28:25,428 INFO [master/f5a5a857f5c5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f5a5a857f5c5,33343,1733300904572-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T08:28:25,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753778e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:28:25,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f5a5a857f5c5,33343,-1 for getting cluster id 2024-12-04T08:28:25,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T08:28:25,438 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '87cabcae-6028-4b29-b095-5793c437b08b' 2024-12-04T08:28:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T08:28:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "87cabcae-6028-4b29-b095-5793c437b08b" 2024-12-04T08:28:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7cc10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:28:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f5a5a857f5c5,33343,-1] 2024-12-04T08:28:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T08:28:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:25,440 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T08:28:25,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f95849a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T08:28:25,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T08:28:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f5a5a857f5c5,46109,1733300904623, seqNum=-1] 2024-12-04T08:28:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T08:28:25,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T08:28:25,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:25,445 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T08:28:25,447 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-04T08:28:25,447 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-04T08:28:25,449 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/test.com,8080,1, archiveDir=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs, maxLogs=32 2024-12-04T08:28:25,449 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733300905449 2024-12-04T08:28:25,454 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/test.com,8080,1/test.com%2C8080%2C1.1733300905449 2024-12-04T08:28:25,454 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40001:40001),(127.0.0.1/127.0.0.1:45527:45527)] 2024-12-04T08:28:25,455 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733300905455 2024-12-04T08:28:25,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,460 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/test.com,8080,1/test.com%2C8080%2C1.1733300905449 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/test.com,8080,1/test.com%2C8080%2C1.1733300905455 2024-12-04T08:28:25,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40001:40001),(127.0.0.1/127.0.0.1:45527:45527)] 2024-12-04T08:28:25,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/test.com,8080,1/test.com%2C8080%2C1.1733300905449 is not closed yet, will try archiving it next time 2024-12-04T08:28:25,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741835_1011 (size=93) 2024-12-04T08:28:25,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741835_1011 (size=93) 2024-12-04T08:28:25,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,462 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/WALs/test.com,8080,1/test.com%2C8080%2C1.1733300905449 to hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs/test.com%2C8080%2C1.1733300905449 2024-12-04T08:28:25,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741836_1012 (size=93) 2024-12-04T08:28:25,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741836_1012 (size=93) 2024-12-04T08:28:25,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs 2024-12-04T08:28:25,465 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733300905455) 2024-12-04T08:28:25,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T08:28:25,465 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:28:25,465 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:28:25,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:25,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:25,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T08:28:25,466 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T08:28:25,466 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1061331260, stopped=false 2024-12-04T08:28:25,466 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f5a5a857f5c5,33343,1733300904572 2024-12-04T08:28:25,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:28:25,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:25,467 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:28:25,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T08:28:25,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:25,467 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T08:28:25,468 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:28:25,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:25,468 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f5a5a857f5c5,46109,1733300904623' ***** 2024-12-04T08:28:25,468 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T08:28:25,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:28:25,468 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T08:28:25,468 INFO [RS:0;f5a5a857f5c5:46109 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T08:28:25,468 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T08:28:25,468 INFO [RS:0;f5a5a857f5c5:46109 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T08:28:25,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T08:28:25,468 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(959): stopping server f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:25,468 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:28:25,468 INFO [RS:0;f5a5a857f5c5:46109 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f5a5a857f5c5:46109. 2024-12-04T08:28:25,468 DEBUG [RS:0;f5a5a857f5c5:46109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T08:28:25,468 DEBUG [RS:0;f5a5a857f5c5:46109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:25,469 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T08:28:25,469 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T08:28:25,469 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T08:28:25,469 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T08:28:25,469 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T08:28:25,469 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T08:28:25,469 DEBUG [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T08:28:25,469 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T08:28:25,469 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T08:28:25,469 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T08:28:25,469 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T08:28:25,469 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T08:28:25,469 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-12-04T08:28:25,484 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/.tmp/ns/692e0f7ab5c944f7baf3c912ee65d96d is 43, key is default/ns:d/1733300905415/Put/seqid=0 2024-12-04T08:28:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741837_1013 (size=5153) 2024-12-04T08:28:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741837_1013 (size=5153) 2024-12-04T08:28:25,489 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/.tmp/ns/692e0f7ab5c944f7baf3c912ee65d96d 2024-12-04T08:28:25,493 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/.tmp/ns/692e0f7ab5c944f7baf3c912ee65d96d as hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/ns/692e0f7ab5c944f7baf3c912ee65d96d 2024-12-04T08:28:25,497 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/ns/692e0f7ab5c944f7baf3c912ee65d96d, entries=2, sequenceid=6, filesize=5.0 K 2024-12-04T08:28:25,498 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=6, compaction requested=false 2024-12-04T08:28:25,501 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T08:28:25,501 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T08:28:25,501 INFO [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T08:28:25,502 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733300905469Running coprocessor pre-close hooks at 1733300905469Disabling compacts and flushes for region at 1733300905469Disabling writes for close at 1733300905469Obtaining lock to block concurrent updates at 1733300905469Preparing flush snapshotting stores in 1588230740 at 1733300905469Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733300905470 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733300905470Flushing 1588230740/ns: creating writer at 1733300905470Flushing 1588230740/ns: appending metadata at 1733300905484 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733300905484Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f687b13: reopening flushed file at 1733300905492 (+8 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=6, compaction requested=false at 1733300905498 (+6 ms)Writing region close event to WAL at 1733300905498Running coprocessor post-close hooks at 1733300905501 (+3 ms)Closed at 1733300905501 2024-12-04T08:28:25,502 DEBUG [RS_CLOSE_META-regionserver/f5a5a857f5c5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T08:28:25,669 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(976): stopping server f5a5a857f5c5,46109,1733300904623; all regions closed. 2024-12-04T08:28:25,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741834_1010 (size=1152) 2024-12-04T08:28:25,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741834_1010 (size=1152) 2024-12-04T08:28:25,674 DEBUG [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs 2024-12-04T08:28:25,674 INFO [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C46109%2C1733300904623.meta:.meta(num 1733300905368) 2024-12-04T08:28:25,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741833_1009 (size=93) 2024-12-04T08:28:25,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741833_1009 (size=93) 2024-12-04T08:28:25,676 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,37469,1733300713386/f5a5a857f5c5%2C37469%2C1733300713386.meta.1733300714278.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:25,676 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36333/user/jenkins/test-data/476e9431-7b28-f577-07a6-3e50f735ab94/WALs/f5a5a857f5c5,35147,1733300714444/f5a5a857f5c5%2C35147%2C1733300714444.1733300714642 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-04T08:28:25,678 DEBUG [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/oldWALs 2024-12-04T08:28:25,678 INFO [RS:0;f5a5a857f5c5:46109 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog f5a5a857f5c5%2C46109%2C1733300904623:(num 1733300904999) 2024-12-04T08:28:25,678 DEBUG [RS:0;f5a5a857f5c5:46109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T08:28:25,678 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T08:28:25,678 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:28:25,678 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.ChoreService(370): Chore service for: regionserver/f5a5a857f5c5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T08:28:25,679 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:28:25,679 INFO [regionserver/f5a5a857f5c5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:28:25,679 INFO [RS:0;f5a5a857f5c5:46109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46109 2024-12-04T08:28:25,681 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:28:25,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T08:28:25,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f5a5a857f5c5,46109,1733300904623 2024-12-04T08:28:25,684 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f5a5a857f5c5,46109,1733300904623] 2024-12-04T08:28:25,685 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f5a5a857f5c5,46109,1733300904623 already deleted, retry=false 2024-12-04T08:28:25,685 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f5a5a857f5c5,46109,1733300904623 expired; onlineServers=0 2024-12-04T08:28:25,685 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f5a5a857f5c5,33343,1733300904572' ***** 2024-12-04T08:28:25,685 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T08:28:25,685 INFO [M:0;f5a5a857f5c5:33343 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T08:28:25,685 INFO [M:0;f5a5a857f5c5:33343 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T08:28:25,685 DEBUG [M:0;f5a5a857f5c5:33343 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T08:28:25,685 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T08:28:25,685 DEBUG [M:0;f5a5a857f5c5:33343 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T08:28:25,685 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300904774 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.large.0-1733300904774,5,FailOnTimeoutGroup] 2024-12-04T08:28:25,685 DEBUG [master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300904774 {}] cleaner.HFileCleaner(306): Exit Thread[master/f5a5a857f5c5:0:becomeActiveMaster-HFileCleaner.small.0-1733300904774,5,FailOnTimeoutGroup] 2024-12-04T08:28:25,686 INFO [M:0;f5a5a857f5c5:33343 {}] hbase.ChoreService(370): Chore service for: master/f5a5a857f5c5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T08:28:25,686 INFO [M:0;f5a5a857f5c5:33343 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T08:28:25,686 DEBUG [M:0;f5a5a857f5c5:33343 {}] master.HMaster(1795): Stopping service threads 2024-12-04T08:28:25,686 INFO [M:0;f5a5a857f5c5:33343 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T08:28:25,686 INFO [M:0;f5a5a857f5c5:33343 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T08:28:25,686 INFO [M:0;f5a5a857f5c5:33343 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T08:28:25,686 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T08:28:25,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T08:28:25,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T08:28:25,687 DEBUG [M:0;f5a5a857f5c5:33343 {}] zookeeper.ZKUtil(347): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T08:28:25,687 WARN [M:0;f5a5a857f5c5:33343 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T08:28:25,687 INFO [M:0;f5a5a857f5c5:33343 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/.lastflushedseqids 2024-12-04T08:28:25,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741838_1014 (size=108) 2024-12-04T08:28:25,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741838_1014 (size=108) 2024-12-04T08:28:25,692 INFO [M:0;f5a5a857f5c5:33343 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T08:28:25,693 INFO [M:0;f5a5a857f5c5:33343 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T08:28:25,693 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T08:28:25,693 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:25,693 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:25,693 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T08:28:25,693 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:25,693 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-12-04T08:28:25,708 DEBUG [M:0;f5a5a857f5c5:33343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f33ebb17ad9d4b7d914c2377a49864b1 is 82, key is hbase:meta,,1/info:regioninfo/1733300905399/Put/seqid=0 2024-12-04T08:28:25,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741839_1015 (size=5672) 2024-12-04T08:28:25,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741839_1015 (size=5672) 2024-12-04T08:28:25,713 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f33ebb17ad9d4b7d914c2377a49864b1 2024-12-04T08:28:25,731 DEBUG [M:0;f5a5a857f5c5:33343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbe6d4f9737e4298ad4266351b1a7e01 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733300905419/Put/seqid=0 2024-12-04T08:28:25,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741840_1016 (size=5275) 2024-12-04T08:28:25,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741840_1016 (size=5275) 2024-12-04T08:28:25,736 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbe6d4f9737e4298ad4266351b1a7e01 2024-12-04T08:28:25,754 DEBUG [M:0;f5a5a857f5c5:33343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e3d4a73523674c2ca8d9761581579ad0 is 69, key is f5a5a857f5c5,46109,1733300904623/rs:state/1733300904857/Put/seqid=0 2024-12-04T08:28:25,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741841_1017 (size=5156) 2024-12-04T08:28:25,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741841_1017 (size=5156) 2024-12-04T08:28:25,759 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e3d4a73523674c2ca8d9761581579ad0 2024-12-04T08:28:25,778 DEBUG [M:0;f5a5a857f5c5:33343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c502a7178daf4cf5937817e43c05ffd5 is 52, key is load_balancer_on/state:d/1733300905446/Put/seqid=0 2024-12-04T08:28:25,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741842_1018 (size=5056) 2024-12-04T08:28:25,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741842_1018 (size=5056) 2024-12-04T08:28:25,782 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c502a7178daf4cf5937817e43c05ffd5 2024-12-04T08:28:25,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:25,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46109-0x1017c96ccfe0001, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:25,784 INFO [RS:0;f5a5a857f5c5:46109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:28:25,784 INFO [RS:0;f5a5a857f5c5:46109 {}] regionserver.HRegionServer(1031): Exiting; stopping=f5a5a857f5c5,46109,1733300904623; zookeeper connection closed. 2024-12-04T08:28:25,784 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@350c2a54 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@350c2a54 2024-12-04T08:28:25,784 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T08:28:25,787 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f33ebb17ad9d4b7d914c2377a49864b1 as hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f33ebb17ad9d4b7d914c2377a49864b1 2024-12-04T08:28:25,790 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f33ebb17ad9d4b7d914c2377a49864b1, entries=8, sequenceid=29, filesize=5.5 K 2024-12-04T08:28:25,791 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbe6d4f9737e4298ad4266351b1a7e01 as hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbe6d4f9737e4298ad4266351b1a7e01 2024-12-04T08:28:25,794 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbe6d4f9737e4298ad4266351b1a7e01, entries=3, sequenceid=29, filesize=5.2 K 2024-12-04T08:28:25,795 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e3d4a73523674c2ca8d9761581579ad0 as hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e3d4a73523674c2ca8d9761581579ad0 2024-12-04T08:28:25,799 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e3d4a73523674c2ca8d9761581579ad0, entries=1, sequenceid=29, filesize=5.0 K 2024-12-04T08:28:25,800 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c502a7178daf4cf5937817e43c05ffd5 as hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c502a7178daf4cf5937817e43c05ffd5 2024-12-04T08:28:25,803 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34681/user/jenkins/test-data/9f5fa28f-1bc1-78c9-bc01-2e161690bdb5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c502a7178daf4cf5937817e43c05ffd5, entries=1, sequenceid=29, filesize=4.9 K 2024-12-04T08:28:25,804 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=29, compaction requested=false 2024-12-04T08:28:25,805 INFO [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T08:28:25,805 DEBUG [M:0;f5a5a857f5c5:33343 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733300905693Disabling compacts and flushes for region at 1733300905693Disabling writes for close at 1733300905693Obtaining lock to block concurrent updates at 1733300905693Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733300905693Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733300905693Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733300905694 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733300905694Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733300905708 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733300905708Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733300905717 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733300905731 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733300905731Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733300905740 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733300905754 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733300905754Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733300905764 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733300905777 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733300905777Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f651c35: reopening flushed file at 1733300905786 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f04a1ac: reopening flushed file at 1733300905790 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6be92c1a: reopening flushed file at 1733300905794 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb3435f: reopening flushed file at 1733300905799 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=29, compaction requested=false at 1733300905804 (+5 ms)Writing region close event to WAL at 1733300905805 (+1 ms)Closed at 1733300905805 2024-12-04T08:28:25,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T08:28:25,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46393 is added to blk_1073741830_1006 (size=10311) 2024-12-04T08:28:25,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43795 is added to blk_1073741830_1006 (size=10311) 2024-12-04T08:28:25,809 INFO [M:0;f5a5a857f5c5:33343 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T08:28:25,809 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T08:28:25,809 INFO [M:0;f5a5a857f5c5:33343 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33343 2024-12-04T08:28:25,809 INFO [M:0;f5a5a857f5c5:33343 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T08:28:25,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:25,919 INFO [M:0;f5a5a857f5c5:33343 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T08:28:25,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33343-0x1017c96ccfe0000, quorum=127.0.0.1:62959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T08:28:25,921 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f9a8217{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:28:25,922 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@35c58925{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:28:25,922 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:28:25,922 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@637d6eb9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:28:25,922 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7748f5df{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir/,STOPPED} 2024-12-04T08:28:25,923 WARN [BP-1517250604-172.17.0.2-1733300903811 heartbeating to localhost/127.0.0.1:34681 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:28:25,923 WARN [BP-1517250604-172.17.0.2-1733300903811 heartbeating to localhost/127.0.0.1:34681 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1517250604-172.17.0.2-1733300903811 (Datanode Uuid 6a263e7a-01ae-4433-8401-f40e78851c33) service to localhost/127.0.0.1:34681 2024-12-04T08:28:25,923 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:28:25,923 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:28:25,924 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data3/current/BP-1517250604-172.17.0.2-1733300903811 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:25,924 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data4/current/BP-1517250604-172.17.0.2-1733300903811 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:25,924 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:28:25,926 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@333ec6ee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T08:28:25,926 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47f122ec{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:28:25,926 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:28:25,926 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76ccace4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:28:25,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e32ebb8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir/,STOPPED} 2024-12-04T08:28:25,928 WARN [BP-1517250604-172.17.0.2-1733300903811 heartbeating to localhost/127.0.0.1:34681 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T08:28:25,928 WARN [BP-1517250604-172.17.0.2-1733300903811 heartbeating to localhost/127.0.0.1:34681 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1517250604-172.17.0.2-1733300903811 (Datanode Uuid 0d50fccb-8ba0-40ca-a3a3-067aa210ee97) service to localhost/127.0.0.1:34681 2024-12-04T08:28:25,928 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T08:28:25,928 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T08:28:25,928 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data1/current/BP-1517250604-172.17.0.2-1733300903811 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:25,929 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/cluster_f47646b6-7078-4c25-e6f2-4a59b0022ad9/data/data2/current/BP-1517250604-172.17.0.2-1733300903811 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T08:28:25,929 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T08:28:25,934 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6dda9ea2{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T08:28:25,935 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a96c77e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T08:28:25,935 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T08:28:25,935 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@eedc0c0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T08:28:25,935 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@512e80eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a289ba33-8072-2d5a-fdc7-3e363f679a43/hadoop.log.dir/,STOPPED} 2024-12-04T08:28:25,941 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T08:28:25,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T08:28:25,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 232) Potentially hanging thread: globalEventExecutor-1-22 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113) app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34681 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34681 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34681 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34681 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34681 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34681 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:34681 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34681 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=541 (was 518) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=13 (was 6) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6346 (was 6353)