2024-12-04 19:26:02,099 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 19:26:02,114 main DEBUG Took 0.012415 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 19:26:02,114 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 19:26:02,115 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 19:26:02,116 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 19:26:02,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,126 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 19:26:02,141 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,145 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,146 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,148 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,150 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,150 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,151 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,152 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,153 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:26:02,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,154 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 19:26:02,156 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:26:02,157 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 19:26:02,159 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 19:26:02,160 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 19:26:02,162 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 19:26:02,162 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 19:26:02,171 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 19:26:02,174 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 19:26:02,176 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 19:26:02,176 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 19:26:02,177 main DEBUG createAppenders(={Console}) 2024-12-04 19:26:02,177 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-04 19:26:02,178 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 19:26:02,178 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-04 19:26:02,179 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 19:26:02,179 main DEBUG OutputStream closed 2024-12-04 19:26:02,179 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 19:26:02,179 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 19:26:02,180 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-04 19:26:02,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 19:26:02,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 19:26:02,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 19:26:02,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 19:26:02,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 19:26:02,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 19:26:02,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 19:26:02,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 19:26:02,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 19:26:02,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 19:26:02,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 19:26:02,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 19:26:02,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 19:26:02,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 19:26:02,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 19:26:02,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 19:26:02,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 19:26:02,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 19:26:02,276 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 19:26:02,276 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-04 19:26:02,276 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 19:26:02,277 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-04T19:26:02,531 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6 2024-12-04 19:26:02,533 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 19:26:02,534 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T19:26:02,545 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-04T19:26:02,553 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-04T19:26:02,579 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T19:26:02,626 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T19:26:02,626 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T19:26:02,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T19:26:02,656 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55, deleteOnExit=true 2024-12-04T19:26:02,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T19:26:02,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/test.cache.data in system properties and HBase conf 2024-12-04T19:26:02,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T19:26:02,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir in system properties and HBase conf 2024-12-04T19:26:02,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T19:26:02,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T19:26:02,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T19:26:02,757 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T19:26:02,871 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T19:26:02,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T19:26:02,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T19:26:02,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T19:26:02,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T19:26:02,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T19:26:02,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T19:26:02,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T19:26:02,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T19:26:02,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T19:26:02,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/nfs.dump.dir in system properties and HBase conf 2024-12-04T19:26:02,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/java.io.tmpdir in system properties and HBase conf 2024-12-04T19:26:02,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T19:26:02,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T19:26:02,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T19:26:04,447 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T19:26:04,538 INFO [Time-limited test {}] log.Log(170): Logging initialized @3091ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T19:26:04,621 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:26:04,687 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:26:04,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:26:04,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:26:04,717 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T19:26:04,734 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:26:04,741 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:26:04,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:26:04,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/java.io.tmpdir/jetty-localhost-41191-hadoop-hdfs-3_4_1-tests_jar-_-any-7891391711384159717/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T19:26:04,932 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:41191} 2024-12-04T19:26:04,933 INFO [Time-limited test {}] server.Server(415): Started @3486ms 2024-12-04T19:26:05,473 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:26:05,482 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:26:05,484 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:26:05,484 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:26:05,484 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T19:26:05,485 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:26:05,485 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:26:05,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/java.io.tmpdir/jetty-localhost-35675-hadoop-hdfs-3_4_1-tests_jar-_-any-1422999635819748844/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:26:05,591 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:35675} 2024-12-04T19:26:05,591 INFO [Time-limited test {}] server.Server(415): Started @4144ms 2024-12-04T19:26:05,633 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:26:05,733 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:26:05,738 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:26:05,739 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:26:05,739 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:26:05,739 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T19:26:05,740 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:26:05,740 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:26:05,854 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/java.io.tmpdir/jetty-localhost-44867-hadoop-hdfs-3_4_1-tests_jar-_-any-2877553303024118765/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:26:05,855 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:44867} 2024-12-04T19:26:05,855 INFO [Time-limited test {}] server.Server(415): Started @4408ms 2024-12-04T19:26:05,858 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:26:05,900 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:26:05,905 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:26:05,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:26:05,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:26:05,909 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T19:26:05,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:26:05,913 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:26:06,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/java.io.tmpdir/jetty-localhost-41047-hadoop-hdfs-3_4_1-tests_jar-_-any-11919740683103674258/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:26:06,014 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:41047} 2024-12-04T19:26:06,014 INFO [Time-limited test {}] server.Server(415): Started @4567ms 2024-12-04T19:26:06,016 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:26:07,393 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data2/current/BP-994869438-172.17.0.2-1733340363677/current, will proceed with Du for space computation calculation, 2024-12-04T19:26:07,393 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data1/current/BP-994869438-172.17.0.2-1733340363677/current, will proceed with Du for space computation calculation, 2024-12-04T19:26:07,396 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data3/current/BP-994869438-172.17.0.2-1733340363677/current, will proceed with Du for space computation calculation, 2024-12-04T19:26:07,396 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data4/current/BP-994869438-172.17.0.2-1733340363677/current, will proceed with Du for space computation calculation, 2024-12-04T19:26:07,426 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:26:07,426 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:26:07,471 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbe2691984d808b69 with lease ID 0xdd0d22f63711fcef: Processing first storage report for DS-7ad38888-fe30-40c5-bb9a-922421de4bec from datanode DatanodeRegistration(127.0.0.1:41073, datanodeUuid=11a6b42d-7789-4a67-9bb1-b5a2934cd644, infoPort=46817, infoSecurePort=0, ipcPort=44731, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677) 2024-12-04T19:26:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbe2691984d808b69 with lease ID 0xdd0d22f63711fcef: from storage DS-7ad38888-fe30-40c5-bb9a-922421de4bec node DatanodeRegistration(127.0.0.1:41073, datanodeUuid=11a6b42d-7789-4a67-9bb1-b5a2934cd644, infoPort=46817, infoSecurePort=0, ipcPort=44731, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T19:26:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb706e3e5e4b24280 with lease ID 0xdd0d22f63711fcee: Processing first storage report for DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c from datanode DatanodeRegistration(127.0.0.1:42863, datanodeUuid=7869fb6a-3980-48db-b24c-8ee77fdbc783, infoPort=34841, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677) 2024-12-04T19:26:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb706e3e5e4b24280 with lease ID 0xdd0d22f63711fcee: from storage DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c node DatanodeRegistration(127.0.0.1:42863, datanodeUuid=7869fb6a-3980-48db-b24c-8ee77fdbc783, infoPort=34841, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:26:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbe2691984d808b69 with lease ID 0xdd0d22f63711fcef: Processing first storage report for DS-2e96e6b5-1487-45af-b527-d3acf86c94d2 from datanode DatanodeRegistration(127.0.0.1:41073, datanodeUuid=11a6b42d-7789-4a67-9bb1-b5a2934cd644, infoPort=46817, infoSecurePort=0, ipcPort=44731, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677) 2024-12-04T19:26:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbe2691984d808b69 with lease ID 0xdd0d22f63711fcef: from storage DS-2e96e6b5-1487-45af-b527-d3acf86c94d2 node DatanodeRegistration(127.0.0.1:41073, datanodeUuid=11a6b42d-7789-4a67-9bb1-b5a2934cd644, infoPort=46817, infoSecurePort=0, ipcPort=44731, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:26:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb706e3e5e4b24280 with lease ID 0xdd0d22f63711fcee: Processing first storage report for DS-54a803ca-2fa0-4554-b23b-071d86091187 from datanode DatanodeRegistration(127.0.0.1:42863, datanodeUuid=7869fb6a-3980-48db-b24c-8ee77fdbc783, infoPort=34841, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677) 2024-12-04T19:26:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb706e3e5e4b24280 with lease ID 0xdd0d22f63711fcee: from storage DS-54a803ca-2fa0-4554-b23b-071d86091187 node DatanodeRegistration(127.0.0.1:42863, datanodeUuid=7869fb6a-3980-48db-b24c-8ee77fdbc783, infoPort=34841, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:26:07,621 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data5/current/BP-994869438-172.17.0.2-1733340363677/current, will proceed with Du for space computation calculation, 2024-12-04T19:26:07,621 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data6/current/BP-994869438-172.17.0.2-1733340363677/current, will proceed with Du for space computation calculation, 2024-12-04T19:26:07,640 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:26:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x274530fd7d5d3e41 with lease ID 0xdd0d22f63711fcf0: Processing first storage report for DS-3f695e16-c624-48eb-9ee2-40e814f34b7a from datanode DatanodeRegistration(127.0.0.1:45577, datanodeUuid=3ed7986c-c0b6-4e2e-b397-5b01cd5417cf, infoPort=39833, infoSecurePort=0, ipcPort=45555, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677) 2024-12-04T19:26:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x274530fd7d5d3e41 with lease ID 0xdd0d22f63711fcf0: from storage DS-3f695e16-c624-48eb-9ee2-40e814f34b7a node DatanodeRegistration(127.0.0.1:45577, datanodeUuid=3ed7986c-c0b6-4e2e-b397-5b01cd5417cf, infoPort=39833, infoSecurePort=0, ipcPort=45555, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:26:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x274530fd7d5d3e41 with lease ID 0xdd0d22f63711fcf0: Processing first storage report for DS-9211ccb8-c2f5-43e1-80d1-5112ef840d25 from datanode DatanodeRegistration(127.0.0.1:45577, datanodeUuid=3ed7986c-c0b6-4e2e-b397-5b01cd5417cf, infoPort=39833, infoSecurePort=0, ipcPort=45555, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677) 2024-12-04T19:26:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x274530fd7d5d3e41 with lease ID 0xdd0d22f63711fcf0: from storage DS-9211ccb8-c2f5-43e1-80d1-5112ef840d25 node DatanodeRegistration(127.0.0.1:45577, datanodeUuid=3ed7986c-c0b6-4e2e-b397-5b01cd5417cf, infoPort=39833, infoSecurePort=0, ipcPort=45555, storageInfo=lv=-57;cid=testClusterID;nsid=167235490;c=1733340363677), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:26:07,715 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6 2024-12-04T19:26:07,790 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/zookeeper_0, clientPort=59927, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T19:26:07,799 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59927 2024-12-04T19:26:07,815 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:07,817 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:07,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:26:07,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:26:07,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:26:08,404 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90 with version=8 2024-12-04T19:26:08,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/hbase-staging 2024-12-04T19:26:08,706 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:26:08,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:08,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:08,726 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:26:08,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:08,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:26:08,871 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T19:26:08,921 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T19:26:08,928 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T19:26:08,932 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:26:08,954 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 70906 (auto-detected) 2024-12-04T19:26:08,955 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T19:26:08,973 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42915 2024-12-04T19:26:08,991 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42915 connecting to ZooKeeper ensemble=127.0.0.1:59927 2024-12-04T19:26:09,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429150x0, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:26:09,098 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42915-0x101a1cf074a0000 connected 2024-12-04T19:26:09,188 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,202 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:09,207 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90, hbase.cluster.distributed=false 2024-12-04T19:26:09,226 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:26:09,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42915 2024-12-04T19:26:09,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42915 2024-12-04T19:26:09,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42915 2024-12-04T19:26:09,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42915 2024-12-04T19:26:09,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42915 2024-12-04T19:26:09,342 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:26:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,344 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:26:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:26:09,347 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:26:09,349 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:26:09,350 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46131 2024-12-04T19:26:09,351 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46131 connecting to ZooKeeper ensemble=127.0.0.1:59927 2024-12-04T19:26:09,352 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461310x0, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:26:09,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461310x0, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:09,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46131-0x101a1cf074a0001 connected 2024-12-04T19:26:09,380 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:26:09,390 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:26:09,394 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:26:09,400 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:26:09,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46131 2024-12-04T19:26:09,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46131 2024-12-04T19:26:09,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46131 2024-12-04T19:26:09,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46131 2024-12-04T19:26:09,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46131 2024-12-04T19:26:09,441 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:26:09,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,442 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:26:09,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:26:09,443 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:26:09,443 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:26:09,445 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41753 2024-12-04T19:26:09,447 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41753 connecting to ZooKeeper ensemble=127.0.0.1:59927 2024-12-04T19:26:09,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,453 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417530x0, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:26:09,470 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417530x0, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:09,471 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:26:09,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41753-0x101a1cf074a0002 connected 2024-12-04T19:26:09,474 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:26:09,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:26:09,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:26:09,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41753 2024-12-04T19:26:09,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41753 2024-12-04T19:26:09,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41753 2024-12-04T19:26:09,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41753 2024-12-04T19:26:09,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41753 2024-12-04T19:26:09,505 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:26:09,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,506 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:26:09,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:26:09,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:26:09,507 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:26:09,507 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:26:09,508 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34937 2024-12-04T19:26:09,510 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34937 connecting to ZooKeeper ensemble=127.0.0.1:59927 2024-12-04T19:26:09,512 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349370x0, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:26:09,529 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:349370x0, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:09,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34937-0x101a1cf074a0003 connected 2024-12-04T19:26:09,530 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:26:09,530 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:26:09,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:26:09,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:26:09,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34937 2024-12-04T19:26:09,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34937 2024-12-04T19:26:09,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34937 2024-12-04T19:26:09,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34937 2024-12-04T19:26:09,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34937 2024-12-04T19:26:09,562 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9ba2734cd39f:42915 2024-12-04T19:26:09,563 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:09,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,583 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:09,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:26:09,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:26:09,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:26:09,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,614 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T19:26:09,615 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9ba2734cd39f,42915,1733340368520 from backup master directory 2024-12-04T19:26:09,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:09,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:26:09,629 WARN [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:26:09,629 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:09,632 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T19:26:09,633 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T19:26:09,695 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/hbase.id] with ID: 5ce29467-e57e-4952-bea7-e9dc4b39f4e6 2024-12-04T19:26:09,695 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/.tmp/hbase.id 2024-12-04T19:26:09,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:26:09,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:26:09,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:26:09,714 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/.tmp/hbase.id]:[hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/hbase.id] 2024-12-04T19:26:09,765 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:26:09,769 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T19:26:09,785 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-04T19:26:09,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:09,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:26:09,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:26:09,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:26:09,828 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:26:09,830 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T19:26:09,834 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:09,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:26:09,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:26:09,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:26:09,876 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store 2024-12-04T19:26:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:26:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:26:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:26:09,900 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T19:26:09,902 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:09,904 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T19:26:09,904 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:09,904 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:09,905 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T19:26:09,905 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:09,906 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:09,906 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733340369903Disabling compacts and flushes for region at 1733340369903Disabling writes for close at 1733340369905 (+2 ms)Writing region close event to WAL at 1733340369905Closed at 1733340369906 (+1 ms) 2024-12-04T19:26:09,908 WARN [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/.initializing 2024-12-04T19:26:09,908 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/WALs/9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:09,915 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:09,928 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C42915%2C1733340368520, suffix=, logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/WALs/9ba2734cd39f,42915,1733340368520, archiveDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/oldWALs, maxLogs=10 2024-12-04T19:26:09,953 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/WALs/9ba2734cd39f,42915,1733340368520/9ba2734cd39f%2C42915%2C1733340368520.1733340369932, exclude list is [], retry=0 2024-12-04T19:26:09,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:09,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:09,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:09,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:09,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T19:26:10,010 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/WALs/9ba2734cd39f,42915,1733340368520/9ba2734cd39f%2C42915%2C1733340368520.1733340369932 2024-12-04T19:26:10,013 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:10,013 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:10,013 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:10,016 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,017 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T19:26:10,078 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:10,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T19:26:10,084 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:10,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T19:26:10,088 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:10,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T19:26:10,092 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:10,094 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,097 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,098 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,102 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,102 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,105 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:26:10,108 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:26:10,112 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:10,113 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67188716, jitterRate=0.001189887523651123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:26:10,119 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733340370030Initializing all the Stores at 1733340370031 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340370032 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340370032Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340370033 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340370033Cleaning up temporary data from old regions at 1733340370102 (+69 ms)Region opened successfully at 1733340370119 (+17 ms) 2024-12-04T19:26:10,120 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T19:26:10,148 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c2a76f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:26:10,182 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T19:26:10,195 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T19:26:10,195 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T19:26:10,198 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T19:26:10,199 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T19:26:10,204 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T19:26:10,205 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T19:26:10,230 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T19:26:10,237 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T19:26:10,290 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T19:26:10,296 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T19:26:10,299 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T19:26:10,310 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T19:26:10,313 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T19:26:10,316 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T19:26:10,327 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T19:26:10,329 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T19:26:10,342 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T19:26:10,363 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T19:26:10,374 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T19:26:10,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:10,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:10,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:10,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:10,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,388 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9ba2734cd39f,42915,1733340368520, sessionid=0x101a1cf074a0000, setting cluster-up flag (Was=false) 2024-12-04T19:26:10,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,448 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T19:26:10,454 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:10,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:10,510 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T19:26:10,512 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:10,520 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T19:26:10,547 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(746): ClusterId : 5ce29467-e57e-4952-bea7-e9dc4b39f4e6 2024-12-04T19:26:10,547 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(746): ClusterId : 5ce29467-e57e-4952-bea7-e9dc4b39f4e6 2024-12-04T19:26:10,547 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(746): ClusterId : 5ce29467-e57e-4952-bea7-e9dc4b39f4e6 2024-12-04T19:26:10,550 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:26:10,550 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:26:10,550 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:26:10,565 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:26:10,565 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:26:10,565 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:26:10,565 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:26:10,565 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:26:10,565 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:26:10,575 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:26:10,575 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:26:10,575 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:26:10,576 DEBUG [RS:2;9ba2734cd39f:34937 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b642432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:26:10,576 DEBUG [RS:1;9ba2734cd39f:41753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6734298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:26:10,576 DEBUG [RS:0;9ba2734cd39f:46131 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281bc6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:26:10,587 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T19:26:10,592 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9ba2734cd39f:34937 2024-12-04T19:26:10,594 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9ba2734cd39f:41753 2024-12-04T19:26:10,594 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9ba2734cd39f:46131 2024-12-04T19:26:10,596 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:26:10,596 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:26:10,596 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:26:10,596 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:26:10,596 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:26:10,596 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:26:10,596 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:26:10,596 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:26:10,596 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:26:10,598 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T19:26:10,599 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,42915,1733340368520 with port=46131, startcode=1733340369309 2024-12-04T19:26:10,599 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,42915,1733340368520 with port=41753, startcode=1733340369441 2024-12-04T19:26:10,600 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,42915,1733340368520 with port=34937, startcode=1733340369505 2024-12-04T19:26:10,606 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T19:26:10,613 DEBUG [RS:1;9ba2734cd39f:41753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:26:10,613 DEBUG [RS:2;9ba2734cd39f:34937 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:26:10,613 DEBUG [RS:0;9ba2734cd39f:46131 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:26:10,613 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9ba2734cd39f,42915,1733340368520 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T19:26:10,622 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:26:10,623 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:26:10,623 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:26:10,623 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:26:10,623 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9ba2734cd39f:0, corePoolSize=10, maxPoolSize=10 2024-12-04T19:26:10,623 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:10,624 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:26:10,624 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:10,651 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733340400651 2024-12-04T19:26:10,653 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:26:10,654 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T19:26:10,655 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T19:26:10,653 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T19:26:10,660 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T19:26:10,660 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47069, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:26:10,660 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:26:10,661 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T19:26:10,661 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33145, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:26:10,661 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T19:26:10,661 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T19:26:10,663 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,663 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T19:26:10,664 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:10,668 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T19:26:10,668 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:26:10,669 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T19:26:10,670 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T19:26:10,672 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T19:26:10,673 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T19:26:10,675 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733340370674,5,FailOnTimeoutGroup] 2024-12-04T19:26:10,675 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733340370675,5,FailOnTimeoutGroup] 2024-12-04T19:26:10,676 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:10,675 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:26:10,676 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T19:26:10,676 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T19:26:10,677 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:10,678 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:10,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:26:10,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:26:10,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:26:10,724 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T19:26:10,725 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90 2024-12-04T19:26:10,733 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T19:26:10,733 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T19:26:10,733 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T19:26:10,734 WARN [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T19:26:10,734 WARN [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T19:26:10,734 WARN [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T19:26:10,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741832_1008 (size=32) 2024-12-04T19:26:10,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741832_1008 (size=32) 2024-12-04T19:26:10,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741832_1008 (size=32) 2024-12-04T19:26:10,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:10,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:26:10,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:26:10,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:10,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:26:10,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:26:10,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:10,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:26:10,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:26:10,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:10,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:26:10,781 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:26:10,782 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:10,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:10,784 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:26:10,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740 2024-12-04T19:26:10,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740 2024-12-04T19:26:10,793 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:26:10,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:26:10,795 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:26:10,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:26:10,803 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:10,804 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66213834, jitterRate=-0.013336986303329468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:26:10,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733340370755Initializing all the Stores at 1733340370757 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340370757Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340370760 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340370760Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340370760Cleaning up temporary data from old regions at 1733340370794 (+34 ms)Region opened successfully at 1733340370807 (+13 ms) 2024-12-04T19:26:10,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:26:10,808 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:26:10,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:26:10,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:26:10,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:26:10,810 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:26:10,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733340370808Disabling compacts and flushes for region at 1733340370808Disabling writes for close at 1733340370808Writing region close event to WAL at 1733340370809 (+1 ms)Closed at 1733340370810 (+1 ms) 2024-12-04T19:26:10,814 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:26:10,814 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T19:26:10,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T19:26:10,828 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:26:10,831 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T19:26:10,835 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,42915,1733340368520 with port=46131, startcode=1733340369309 2024-12-04T19:26:10,835 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,42915,1733340368520 with port=41753, startcode=1733340369441 2024-12-04T19:26:10,835 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,42915,1733340368520 with port=34937, startcode=1733340369505 2024-12-04T19:26:10,837 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:10,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:10,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:10,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:10,847 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90 2024-12-04T19:26:10,847 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33061 2024-12-04T19:26:10,847 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:26:10,849 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:10,849 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90 2024-12-04T19:26:10,849 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:10,849 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33061 2024-12-04T19:26:10,850 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:26:10,852 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90 2024-12-04T19:26:10,852 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33061 2024-12-04T19:26:10,852 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:26:10,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:26:10,982 WARN [9ba2734cd39f:42915 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T19:26:10,995 DEBUG [RS:1;9ba2734cd39f:41753 {}] zookeeper.ZKUtil(111): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:10,996 WARN [RS:1;9ba2734cd39f:41753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:26:10,996 INFO [RS:1;9ba2734cd39f:41753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:10,996 DEBUG [RS:2;9ba2734cd39f:34937 {}] zookeeper.ZKUtil(111): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:10,996 DEBUG [RS:0;9ba2734cd39f:46131 {}] zookeeper.ZKUtil(111): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:10,996 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:10,996 WARN [RS:2;9ba2734cd39f:34937 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:26:10,997 WARN [RS:0;9ba2734cd39f:46131 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:26:10,997 INFO [RS:2;9ba2734cd39f:34937 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:10,997 INFO [RS:0;9ba2734cd39f:46131 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:10,997 DEBUG [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:10,997 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:10,999 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,46131,1733340369309] 2024-12-04T19:26:10,999 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,34937,1733340369505] 2024-12-04T19:26:10,999 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,41753,1733340369441] 2024-12-04T19:26:11,023 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:26:11,023 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:26:11,023 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:26:11,037 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:26:11,037 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:26:11,037 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:26:11,043 INFO [RS:2;9ba2734cd39f:34937 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:26:11,043 INFO [RS:0;9ba2734cd39f:46131 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:26:11,043 INFO [RS:1;9ba2734cd39f:41753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:26:11,044 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,044 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,044 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,045 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:26:11,049 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:26:11,051 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:26:11,051 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:26:11,052 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:26:11,053 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,053 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,054 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:26:11,054 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,054 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,054 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:26:11,054 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:26:11,055 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:26:11,055 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:26:11,055 DEBUG [RS:2;9ba2734cd39f:34937 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:26:11,056 DEBUG [RS:1;9ba2734cd39f:41753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:26:11,056 DEBUG [RS:0;9ba2734cd39f:46131 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:26:11,063 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,41753,1733340369441-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,34937,1733340369505-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:26:11,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46131,1733340369309-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:26:11,085 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:26:11,085 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:26:11,087 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:26:11,087 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46131,1733340369309-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,087 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,34937,1733340369505-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,087 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,41753,1733340369441-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,088 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,088 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,088 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,088 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.Replication(171): 9ba2734cd39f,46131,1733340369309 started 2024-12-04T19:26:11,088 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.Replication(171): 9ba2734cd39f,41753,1733340369441 started 2024-12-04T19:26:11,088 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.Replication(171): 9ba2734cd39f,34937,1733340369505 started 2024-12-04T19:26:11,108 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,108 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,108 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,41753,1733340369441, RpcServer on 9ba2734cd39f/172.17.0.2:41753, sessionid=0x101a1cf074a0002 2024-12-04T19:26:11,108 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,34937,1733340369505, RpcServer on 9ba2734cd39f/172.17.0.2:34937, sessionid=0x101a1cf074a0003 2024-12-04T19:26:11,109 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:26:11,109 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:26:11,109 DEBUG [RS:1;9ba2734cd39f:41753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:11,109 DEBUG [RS:2;9ba2734cd39f:34937 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:11,109 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,34937,1733340369505' 2024-12-04T19:26:11,109 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,41753,1733340369441' 2024-12-04T19:26:11,110 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:26:11,110 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:26:11,110 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:26:11,110 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:26:11,111 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,111 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,46131,1733340369309, RpcServer on 9ba2734cd39f/172.17.0.2:46131, sessionid=0x101a1cf074a0001 2024-12-04T19:26:11,111 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:26:11,111 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:26:11,111 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:26:11,111 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:26:11,111 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:26:11,111 DEBUG [RS:0;9ba2734cd39f:46131 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:11,111 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,46131,1733340369309' 2024-12-04T19:26:11,111 DEBUG [RS:2;9ba2734cd39f:34937 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:11,111 DEBUG [RS:1;9ba2734cd39f:41753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:11,111 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:26:11,111 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,41753,1733340369441' 2024-12-04T19:26:11,111 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,34937,1733340369505' 2024-12-04T19:26:11,111 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:26:11,111 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:26:11,112 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:26:11,112 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:26:11,112 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:26:11,112 DEBUG [RS:2;9ba2734cd39f:34937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:26:11,112 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:26:11,112 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:26:11,112 DEBUG [RS:1;9ba2734cd39f:41753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:26:11,112 INFO [RS:2;9ba2734cd39f:34937 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:26:11,112 DEBUG [RS:0;9ba2734cd39f:46131 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:11,112 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,46131,1733340369309' 2024-12-04T19:26:11,112 INFO [RS:1;9ba2734cd39f:41753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:26:11,113 INFO [RS:2;9ba2734cd39f:34937 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:26:11,113 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:26:11,113 INFO [RS:1;9ba2734cd39f:41753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:26:11,113 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:26:11,114 DEBUG [RS:0;9ba2734cd39f:46131 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:26:11,114 INFO [RS:0;9ba2734cd39f:46131 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:26:11,114 INFO [RS:0;9ba2734cd39f:46131 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:26:11,219 INFO [RS:0;9ba2734cd39f:46131 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:11,219 INFO [RS:2;9ba2734cd39f:34937 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:11,219 INFO [RS:1;9ba2734cd39f:41753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:11,222 INFO [RS:0;9ba2734cd39f:46131 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C46131%2C1733340369309, suffix=, logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309, archiveDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs, maxLogs=32 2024-12-04T19:26:11,222 INFO [RS:2;9ba2734cd39f:34937 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C34937%2C1733340369505, suffix=, logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,34937,1733340369505, archiveDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs, maxLogs=32 2024-12-04T19:26:11,222 INFO [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C41753%2C1733340369441, suffix=, logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441, archiveDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs, maxLogs=32 2024-12-04T19:26:11,238 DEBUG [RS:2;9ba2734cd39f:34937 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,34937,1733340369505/9ba2734cd39f%2C34937%2C1733340369505.1733340371226, exclude list is [], retry=0 2024-12-04T19:26:11,238 DEBUG [RS:1;9ba2734cd39f:41753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441/9ba2734cd39f%2C41753%2C1733340369441.1733340371226, exclude list is [], retry=0 2024-12-04T19:26:11,238 DEBUG [RS:0;9ba2734cd39f:46131 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309/9ba2734cd39f%2C46131%2C1733340369309.1733340371226, exclude list is [], retry=0 2024-12-04T19:26:11,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:11,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:11,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:11,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:11,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:11,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:11,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:11,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:11,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:11,249 INFO [RS:0;9ba2734cd39f:46131 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 2024-12-04T19:26:11,250 DEBUG [RS:0;9ba2734cd39f:46131 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:11,251 INFO [RS:2;9ba2734cd39f:34937 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,34937,1733340369505/9ba2734cd39f%2C34937%2C1733340369505.1733340371226 2024-12-04T19:26:11,252 DEBUG [RS:2;9ba2734cd39f:34937 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:11,252 INFO [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441/9ba2734cd39f%2C41753%2C1733340369441.1733340371226 2024-12-04T19:26:11,256 DEBUG [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:11,485 DEBUG [9ba2734cd39f:42915 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-04T19:26:11,493 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:26:11,500 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:26:11,501 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:26:11,501 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T19:26:11,501 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:26:11,501 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:26:11,501 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T19:26:11,501 INFO [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:26:11,501 INFO [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:26:11,501 INFO [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T19:26:11,501 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:26:11,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:11,512 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,41753,1733340369441, state=OPENING 2024-12-04T19:26:11,563 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T19:26:11,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:11,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:11,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:11,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:11,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,577 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:26:11,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,41753,1733340369441}] 2024-12-04T19:26:11,763 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:26:11,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36975, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:26:11,776 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T19:26:11,776 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:11,777 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T19:26:11,780 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C41753%2C1733340369441.meta, suffix=.meta, logDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441, archiveDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs, maxLogs=32 2024-12-04T19:26:11,796 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441/9ba2734cd39f%2C41753%2C1733340369441.meta.1733340371782.meta, exclude list is [], retry=0 2024-12-04T19:26:11,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:11,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:11,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:11,803 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,41753,1733340369441/9ba2734cd39f%2C41753%2C1733340369441.meta.1733340371782.meta 2024-12-04T19:26:11,804 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:11,804 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:11,806 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T19:26:11,809 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T19:26:11,814 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T19:26:11,819 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T19:26:11,819 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:11,819 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T19:26:11,819 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T19:26:11,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:26:11,824 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:26:11,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:11,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:11,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:26:11,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:26:11,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:11,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:11,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:26:11,828 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:26:11,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:11,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:11,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:26:11,831 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:26:11,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:11,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:26:11,832 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:26:11,833 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740 2024-12-04T19:26:11,836 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740 2024-12-04T19:26:11,839 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:26:11,839 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:26:11,840 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:26:11,842 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:26:11,844 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70500701, jitterRate=0.05054230988025665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:26:11,844 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T19:26:11,846 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733340371820Writing region info on filesystem at 1733340371820Initializing all the Stores at 1733340371822 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340371822Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340371822Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340371822Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340371822Cleaning up temporary data from old regions at 1733340371839 (+17 ms)Running coprocessor post-open hooks at 1733340371844 (+5 ms)Region opened successfully at 1733340371846 (+2 ms) 2024-12-04T19:26:11,853 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733340371756 2024-12-04T19:26:11,863 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T19:26:11,863 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T19:26:11,865 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:11,866 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,41753,1733340369441, state=OPEN 2024-12-04T19:26:11,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:26:11,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:26:11,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:26:11,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:26:11,879 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,879 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,879 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,879 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:26:11,880 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:11,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T19:26:11,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,41753,1733340369441 in 301 msec 2024-12-04T19:26:11,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T19:26:11,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0680 sec 2024-12-04T19:26:11,896 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:26:11,896 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T19:26:11,916 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:26:11,917 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,41753,1733340369441, seqNum=-1] 2024-12-04T19:26:11,934 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:26:11,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52073, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:26:11,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4070 sec 2024-12-04T19:26:11,955 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733340371955, completionTime=-1 2024-12-04T19:26:11,959 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-04T19:26:11,959 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T19:26:11,985 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-04T19:26:11,986 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733340431986 2024-12-04T19:26:11,986 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733340491986 2024-12-04T19:26:11,986 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-04T19:26:11,987 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T19:26:11,996 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42915,1733340368520-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,996 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42915,1733340368520-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,996 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42915,1733340368520-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,998 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9ba2734cd39f:42915, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,999 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:11,999 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:12,007 DEBUG [master/9ba2734cd39f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T19:26:12,064 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.435sec 2024-12-04T19:26:12,073 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T19:26:12,074 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T19:26:12,076 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T19:26:12,076 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T19:26:12,077 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T19:26:12,078 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42915,1733340368520-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:26:12,079 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42915,1733340368520-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T19:26:12,089 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T19:26:12,091 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T19:26:12,092 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42915,1733340368520-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:12,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78619a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:26:12,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,42915,-1 for getting cluster id 2024-12-04T19:26:12,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:26:12,172 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5ce29467-e57e-4952-bea7-e9dc4b39f4e6' 2024-12-04T19:26:12,174 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:26:12,174 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5ce29467-e57e-4952-bea7-e9dc4b39f4e6" 2024-12-04T19:26:12,175 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa37995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:26:12,175 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,42915,-1] 2024-12-04T19:26:12,178 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:26:12,180 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:12,181 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:26:12,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd02f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:26:12,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:26:12,193 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,41753,1733340369441, seqNum=-1] 2024-12-04T19:26:12,194 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:26:12,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:26:12,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:12,226 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33061/hbase 2024-12-04T19:26:12,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=16264 2024-12-04T19:26:12,268 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:12,271 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:12,272 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:12,276 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58218494, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-58218494, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:12,289 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58218494/hregion-58218494.1733340372278, exclude list is [], retry=0 2024-12-04T19:26:12,293 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:12,293 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:12,294 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:12,297 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58218494/hregion-58218494.1733340372278 2024-12-04T19:26:12,298 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:12,298 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0b16d1b7e8b5ee9efa5e20cdae3c82b6, NAME => 'testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741838_1014 (size=64) 2024-12-04T19:26:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741838_1014 (size=64) 2024-12-04T19:26:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741838_1014 (size=64) 2024-12-04T19:26:12,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:12,312 INFO [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,314 INFO [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b16d1b7e8b5ee9efa5e20cdae3c82b6 columnFamilyName a 2024-12-04T19:26:12,314 DEBUG [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:12,315 INFO [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] regionserver.HStore(327): Store=0b16d1b7e8b5ee9efa5e20cdae3c82b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,316 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,318 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,318 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,320 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,323 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:12,324 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0b16d1b7e8b5ee9efa5e20cdae3c82b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64263928, jitterRate=-0.042392849922180176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:12,325 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0b16d1b7e8b5ee9efa5e20cdae3c82b6: Writing region info on filesystem at 1733340372311Initializing all the Stores at 1733340372312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340372312Cleaning up temporary data from old regions at 1733340372318 (+6 ms)Region opened successfully at 1733340372325 (+7 ms) 2024-12-04T19:26:12,325 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0b16d1b7e8b5ee9efa5e20cdae3c82b6, disabling compactions & flushes 2024-12-04T19:26:12,325 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6. 2024-12-04T19:26:12,325 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6. 2024-12-04T19:26:12,325 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6. after waiting 0 ms 2024-12-04T19:26:12,325 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6. 2024-12-04T19:26:12,325 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6. 2024-12-04T19:26:12,325 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0b16d1b7e8b5ee9efa5e20cdae3c82b6: Waiting for close lock at 1733340372325Disabling compacts and flushes for region at 1733340372325Disabling writes for close at 1733340372325Writing region close event to WAL at 1733340372325Closed at 1733340372325 2024-12-04T19:26:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741837_1013 (size=93) 2024-12-04T19:26:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741837_1013 (size=93) 2024-12-04T19:26:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741837_1013 (size=93) 2024-12-04T19:26:12,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:12,337 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58218494:(num 1733340372278) 2024-12-04T19:26:12,338 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:26:12,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741839_1015 (size=276) 2024-12-04T19:26:12,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741839_1015 (size=276) 2024-12-04T19:26:12,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741839_1015 (size=276) 2024-12-04T19:26:12,357 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:26:12,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741840_1016 (size=230) 2024-12-04T19:26:12,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741840_1016 (size=230) 2024-12-04T19:26:12,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741840_1016 (size=230) 2024-12-04T19:26:12,387 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1, size=276 (276bytes) 2024-12-04T19:26:12,388 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T19:26:12,388 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T19:26:12,388 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1 2024-12-04T19:26:12,393 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1 after 3ms 2024-12-04T19:26:12,398 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,399 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1 took 13ms 2024-12-04T19:26:12,402 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1 so closing down 2024-12-04T19:26:12,405 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T19:26:12,407 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T19:26:12,407 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:12,408 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741841_1017 (size=276) 2024-12-04T19:26:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741841_1017 (size=276) 2024-12-04T19:26:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741841_1017 (size=276) 2024-12-04T19:26:12,418 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:12,421 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002 2024-12-04T19:26:12,424 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-04T19:26:12,424 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1, journal: Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1, size=276 (276bytes) at 1733340372387Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1 so closing down at 1733340372402 (+15 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000001-wal-1.temp at 1733340372407 (+5 ms)3 split writer threads finished at 1733340372408 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733340372418 (+10 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002 at 1733340372421 (+3 ms)Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733340372424 (+3 ms) 2024-12-04T19:26:12,437 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2, size=230 (230bytes) 2024-12-04T19:26:12,437 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2 2024-12-04T19:26:12,438 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2 after 1ms 2024-12-04T19:26:12,441 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,441 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2 took 4ms 2024-12-04T19:26:12,443 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2 so closing down 2024-12-04T19:26:12,443 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:12,446 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T19:26:12,447 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T19:26:12,447 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:12,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741842_1018 (size=230) 2024-12-04T19:26:12,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741842_1018 (size=230) 2024-12-04T19:26:12,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741842_1018 (size=230) 2024-12-04T19:26:12,458 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:12,462 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,465 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-04T19:26:12,466 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-04T19:26:12,466 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2, journal: Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2, size=230 (230bytes) at 1733340372437Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2 so closing down at 1733340372443 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002-wal-2.temp at 1733340372447 (+4 ms)3 split writer threads finished at 1733340372447Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733340372458 (+11 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733340372466 (+8 ms) 2024-12-04T19:26:12,467 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:12,469 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:12,482 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal.1733340372470, exclude list is [], retry=0 2024-12-04T19:26:12,486 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:12,487 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:12,487 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:12,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal.1733340372470 2024-12-04T19:26:12,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:12,490 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b16d1b7e8b5ee9efa5e20cdae3c82b6, NAME => 'testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:12,490 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:12,490 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,490 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,492 INFO [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,494 INFO [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b16d1b7e8b5ee9efa5e20cdae3c82b6 columnFamilyName a 2024-12-04T19:26:12,494 DEBUG [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:12,495 INFO [StoreOpener-0b16d1b7e8b5ee9efa5e20cdae3c82b6-1 {}] regionserver.HStore(327): Store=0b16d1b7e8b5ee9efa5e20cdae3c82b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:12,495 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,499 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,500 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002 2024-12-04T19:26:12,503 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,508 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002 2024-12-04T19:26:12,511 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0b16d1b7e8b5ee9efa5e20cdae3c82b6 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T19:26:12,556 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/.tmp/a/8abd9776d10247caa9aaac29424f7ec7 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733340372337/Put/seqid=0 2024-12-04T19:26:12,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741844_1020 (size=5170) 2024-12-04T19:26:12,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741844_1020 (size=5170) 2024-12-04T19:26:12,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741844_1020 (size=5170) 2024-12-04T19:26:12,571 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/.tmp/a/8abd9776d10247caa9aaac29424f7ec7 2024-12-04T19:26:12,608 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/.tmp/a/8abd9776d10247caa9aaac29424f7ec7 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/a/8abd9776d10247caa9aaac29424f7ec7 2024-12-04T19:26:12,619 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/a/8abd9776d10247caa9aaac29424f7ec7, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T19:26:12,624 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 in 111ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T19:26:12,626 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/0000000000000000002 2024-12-04T19:26:12,627 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,627 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,630 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 2024-12-04T19:26:12,634 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/0b16d1b7e8b5ee9efa5e20cdae3c82b6/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T19:26:12,636 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0b16d1b7e8b5ee9efa5e20cdae3c82b6; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66604394, jitterRate=-0.007517188787460327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:12,637 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0b16d1b7e8b5ee9efa5e20cdae3c82b6: Writing region info on filesystem at 1733340372491Initializing all the Stores at 1733340372492 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340372492Obtaining lock to block concurrent updates at 1733340372511 (+19 ms)Preparing flush snapshotting stores in 0b16d1b7e8b5ee9efa5e20cdae3c82b6 at 1733340372511Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733340372515 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733340372269.0b16d1b7e8b5ee9efa5e20cdae3c82b6. at 1733340372515Flushing 0b16d1b7e8b5ee9efa5e20cdae3c82b6/a: creating writer at 1733340372516 (+1 ms)Flushing 0b16d1b7e8b5ee9efa5e20cdae3c82b6/a: appending metadata at 1733340372546 (+30 ms)Flushing 0b16d1b7e8b5ee9efa5e20cdae3c82b6/a: closing flushed file at 1733340372548 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b33c23f: reopening flushed file at 1733340372606 (+58 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0b16d1b7e8b5ee9efa5e20cdae3c82b6 in 111ms, sequenceid=2, compaction requested=false; wal=null at 1733340372624 (+18 ms)Cleaning up temporary data from old regions at 1733340372627 (+3 ms)Region opened successfully at 1733340372637 (+10 ms) 2024-12-04T19:26:12,667 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=378 (was 366) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:55618 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33061/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43528 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:33790 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43546 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54052 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:41572 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=16251 (was 16264) 2024-12-04T19:26:12,682 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=16247 2024-12-04T19:26:12,699 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:12,701 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:12,703 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:12,707 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95093230, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-95093230, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:12,721 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95093230/hregion-95093230.1733340372708, exclude list is [], retry=0 2024-12-04T19:26:12,725 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:12,725 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:12,726 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:12,728 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95093230/hregion-95093230.1733340372708 2024-12-04T19:26:12,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:12,734 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 629abe42c517f2dcd191ff9af86c2d97, NAME => 'testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:12,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741846_1022 (size=64) 2024-12-04T19:26:12,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741846_1022 (size=64) 2024-12-04T19:26:12,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741846_1022 (size=64) 2024-12-04T19:26:12,751 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:12,753 INFO [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,756 INFO [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629abe42c517f2dcd191ff9af86c2d97 columnFamilyName a 2024-12-04T19:26:12,756 DEBUG [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:12,757 INFO [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] regionserver.HStore(327): Store=629abe42c517f2dcd191ff9af86c2d97/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:12,757 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,759 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,759 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,760 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,760 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,764 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,768 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:12,769 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 629abe42c517f2dcd191ff9af86c2d97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66954414, jitterRate=-0.0023014843463897705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:12,769 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 629abe42c517f2dcd191ff9af86c2d97: Writing region info on filesystem at 1733340372751Initializing all the Stores at 1733340372753 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340372753Cleaning up temporary data from old regions at 1733340372760 (+7 ms)Region opened successfully at 1733340372769 (+9 ms) 2024-12-04T19:26:12,769 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 629abe42c517f2dcd191ff9af86c2d97, disabling compactions & flushes 2024-12-04T19:26:12,769 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97. 2024-12-04T19:26:12,769 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97. 2024-12-04T19:26:12,769 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97. after waiting 0 ms 2024-12-04T19:26:12,769 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97. 2024-12-04T19:26:12,770 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97. 2024-12-04T19:26:12,770 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 629abe42c517f2dcd191ff9af86c2d97: Waiting for close lock at 1733340372769Disabling compacts and flushes for region at 1733340372769Disabling writes for close at 1733340372769Writing region close event to WAL at 1733340372770 (+1 ms)Closed at 1733340372770 2024-12-04T19:26:12,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741845_1021 (size=93) 2024-12-04T19:26:12,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741845_1021 (size=93) 2024-12-04T19:26:12,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741845_1021 (size=93) 2024-12-04T19:26:12,777 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:12,778 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95093230:(num 1733340372708) 2024-12-04T19:26:12,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:26:12,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741847_1023 (size=276) 2024-12-04T19:26:12,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741847_1023 (size=276) 2024-12-04T19:26:12,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741847_1023 (size=276) 2024-12-04T19:26:12,795 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:26:12,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741848_1024 (size=230) 2024-12-04T19:26:12,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741848_1024 (size=230) 2024-12-04T19:26:12,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741848_1024 (size=230) 2024-12-04T19:26:12,822 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2, size=230 (230bytes) 2024-12-04T19:26:12,822 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2 2024-12-04T19:26:12,823 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2 after 1ms 2024-12-04T19:26:12,827 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,827 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2 took 5ms 2024-12-04T19:26:12,829 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2 so closing down 2024-12-04T19:26:12,829 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:12,831 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T19:26:12,832 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T19:26:12,833 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:12,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741849_1025 (size=230) 2024-12-04T19:26:12,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741849_1025 (size=230) 2024-12-04T19:26:12,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741849_1025 (size=230) 2024-12-04T19:26:12,841 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:12,843 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 2024-12-04T19:26:12,843 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-04T19:26:12,844 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2, journal: Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2, size=230 (230bytes) at 1733340372822Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2 so closing down at 1733340372829 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002-wal-2.temp at 1733340372833 (+4 ms)3 split writer threads finished at 1733340372833Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733340372841 (+8 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 at 1733340372843 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733340372843 2024-12-04T19:26:12,856 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1, size=276 (276bytes) 2024-12-04T19:26:12,856 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1 2024-12-04T19:26:12,857 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1 after 1ms 2024-12-04T19:26:12,860 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,860 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1 took 4ms 2024-12-04T19:26:12,862 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1 so closing down 2024-12-04T19:26:12,863 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:12,865 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T19:26:12,867 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T19:26:12,867 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:12,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741850_1026 (size=276) 2024-12-04T19:26:12,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741850_1026 (size=276) 2024-12-04T19:26:12,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741850_1026 (size=276) 2024-12-04T19:26:12,877 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:12,881 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,883 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002, length=230 2024-12-04T19:26:12,885 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 2024-12-04T19:26:12,885 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-04T19:26:12,886 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1, journal: Splitting hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1, size=276 (276bytes) at 1733340372856Finishing writing output for hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1 so closing down at 1733340372863 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000001-wal-1.temp at 1733340372867 (+4 ms)3 split writer threads finished at 1733340372867Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733340372877 (+10 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 at 1733340372885 (+8 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733340372885 2024-12-04T19:26:12,886 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:12,889 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:12,904 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal.1733340372889, exclude list is [], retry=0 2024-12-04T19:26:12,907 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:12,908 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:12,908 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:12,910 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal.1733340372889 2024-12-04T19:26:12,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:12,911 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 629abe42c517f2dcd191ff9af86c2d97, NAME => 'testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:12,911 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:12,911 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,911 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,913 INFO [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,915 INFO [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629abe42c517f2dcd191ff9af86c2d97 columnFamilyName a 2024-12-04T19:26:12,915 DEBUG [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:12,916 INFO [StoreOpener-629abe42c517f2dcd191ff9af86c2d97-1 {}] regionserver.HStore(327): Store=629abe42c517f2dcd191ff9af86c2d97/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:12,916 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,917 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,920 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,921 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 2024-12-04T19:26:12,924 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:12,926 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 2024-12-04T19:26:12,926 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 629abe42c517f2dcd191ff9af86c2d97 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T19:26:12,943 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/.tmp/a/f0f68637c47b43119cc997dfc5f9a0b4 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733340372778/Put/seqid=0 2024-12-04T19:26:12,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741852_1028 (size=5170) 2024-12-04T19:26:12,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741852_1028 (size=5170) 2024-12-04T19:26:12,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741852_1028 (size=5170) 2024-12-04T19:26:12,953 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/.tmp/a/f0f68637c47b43119cc997dfc5f9a0b4 2024-12-04T19:26:12,962 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/.tmp/a/f0f68637c47b43119cc997dfc5f9a0b4 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/a/f0f68637c47b43119cc997dfc5f9a0b4 2024-12-04T19:26:12,969 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/a/f0f68637c47b43119cc997dfc5f9a0b4, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T19:26:12,970 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 629abe42c517f2dcd191ff9af86c2d97 in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T19:26:12,971 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/0000000000000000002 2024-12-04T19:26:12,971 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,971 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,974 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 629abe42c517f2dcd191ff9af86c2d97 2024-12-04T19:26:12,977 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/629abe42c517f2dcd191ff9af86c2d97/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T19:26:12,978 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 629abe42c517f2dcd191ff9af86c2d97; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60301160, jitterRate=-0.10144269466400146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:12,978 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 629abe42c517f2dcd191ff9af86c2d97: Writing region info on filesystem at 1733340372911Initializing all the Stores at 1733340372913 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340372913Obtaining lock to block concurrent updates at 1733340372927 (+14 ms)Preparing flush snapshotting stores in 629abe42c517f2dcd191ff9af86c2d97 at 1733340372927Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733340372927Flushing stores of testReplayEditsWrittenIntoWAL,,1733340372700.629abe42c517f2dcd191ff9af86c2d97. at 1733340372927Flushing 629abe42c517f2dcd191ff9af86c2d97/a: creating writer at 1733340372927Flushing 629abe42c517f2dcd191ff9af86c2d97/a: appending metadata at 1733340372942 (+15 ms)Flushing 629abe42c517f2dcd191ff9af86c2d97/a: closing flushed file at 1733340372942Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c80898c: reopening flushed file at 1733340372960 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 629abe42c517f2dcd191ff9af86c2d97 in 43ms, sequenceid=2, compaction requested=false; wal=null at 1733340372970 (+10 ms)Cleaning up temporary data from old regions at 1733340372971 (+1 ms)Region opened successfully at 1733340372978 (+7 ms) 2024-12-04T19:26:12,995 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43640 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:41572 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54128 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43528 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:55698 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:33790 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=16239 (was 16247) 2024-12-04T19:26:13,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=16238 2024-12-04T19:26:13,021 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:13,024 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:13,025 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:13,028 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91134287, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-91134287, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:13,038 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91134287/hregion-91134287.1733340373028, exclude list is [], retry=0 2024-12-04T19:26:13,042 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:13,042 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:13,042 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:13,045 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91134287/hregion-91134287.1733340373028 2024-12-04T19:26:13,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:13,045 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3ffbd393a28d04d387295a767ed41401, NAME => 'testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741854_1030 (size=64) 2024-12-04T19:26:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741854_1030 (size=64) 2024-12-04T19:26:13,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741854_1030 (size=64) 2024-12-04T19:26:13,057 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:13,058 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,060 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ffbd393a28d04d387295a767ed41401 columnFamilyName a 2024-12-04T19:26:13,060 DEBUG [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:13,061 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(327): Store=3ffbd393a28d04d387295a767ed41401/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:13,061 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,062 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ffbd393a28d04d387295a767ed41401 columnFamilyName b 2024-12-04T19:26:13,062 DEBUG [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:13,063 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(327): Store=3ffbd393a28d04d387295a767ed41401/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:13,063 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,064 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ffbd393a28d04d387295a767ed41401 columnFamilyName c 2024-12-04T19:26:13,065 DEBUG [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:13,065 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(327): Store=3ffbd393a28d04d387295a767ed41401/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:13,065 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,067 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,067 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,068 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:13,069 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:13,071 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:13,072 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ffbd393a28d04d387295a767ed41401; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72962933, jitterRate=0.08723242580890656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:13,072 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ffbd393a28d04d387295a767ed41401: Writing region info on filesystem at 1733340373057Initializing all the Stores at 1733340373058 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340373058Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340373058Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340373058Cleaning up temporary data from old regions at 1733340373067 (+9 ms)Region opened successfully at 1733340373072 (+5 ms) 2024-12-04T19:26:13,073 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ffbd393a28d04d387295a767ed41401, disabling compactions & flushes 2024-12-04T19:26:13,073 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:13,073 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:13,073 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. after waiting 0 ms 2024-12-04T19:26:13,073 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:13,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:13,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ffbd393a28d04d387295a767ed41401: Waiting for close lock at 1733340373072Disabling compacts and flushes for region at 1733340373072Disabling writes for close at 1733340373073 (+1 ms)Writing region close event to WAL at 1733340373073Closed at 1733340373073 2024-12-04T19:26:13,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741853_1029 (size=93) 2024-12-04T19:26:13,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741853_1029 (size=93) 2024-12-04T19:26:13,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741853_1029 (size=93) 2024-12-04T19:26:13,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:13,080 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91134287:(num 1733340373028) 2024-12-04T19:26:13,080 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:13,082 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:13,094 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083, exclude list is [], retry=0 2024-12-04T19:26:13,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:13,098 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:13,098 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:13,101 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 2024-12-04T19:26:13,102 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:13,314 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083, size=0 (0bytes) 2024-12-04T19:26:13,314 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 might be still open, length is 0 2024-12-04T19:26:13,314 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 2024-12-04T19:26:13,315 WARN [IPC Server handler 1 on default port 33061 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-04T19:26:13,316 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 after 2ms 2024-12-04T19:26:13,661 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43692 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43692 dst: /127.0.0.1:42863 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42863 remote=/127.0.0.1:43692]. Total timeout mills is 60000, 59624 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:13,663 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:55720 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55720 dst: /127.0.0.1:45577 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:13,663 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54164 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41073:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54164 dst: /127.0.0.1:41073 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:13,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741855_1032 (size=200589) 2024-12-04T19:26:13,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741855_1032 (size=200589) 2024-12-04T19:26:13,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741855_1032 (size=200589) 2024-12-04T19:26:17,317 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 after 4003ms 2024-12-04T19:26:17,321 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:17,322 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 took 4008ms 2024-12-04T19:26:17,327 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733340373083.temp 2024-12-04T19:26:17,331 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:26:17,334 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000000001-wal.1733340373083.temp 2024-12-04T19:26:17,407 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T19:26:17,450 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083; continuing. 2024-12-04T19:26:17,451 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 so closing down 2024-12-04T19:26:17,451 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:17,451 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:17,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741856_1033 (size=200597) 2024-12-04T19:26:17,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741856_1033 (size=200597) 2024-12-04T19:26:17,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741856_1033 (size=200597) 2024-12-04T19:26:17,457 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000000001-wal.1733340373083.temp (wrote 3002 edits, skipped 0 edits in 47 ms) 2024-12-04T19:26:17,459 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000000001-wal.1733340373083.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002 2024-12-04T19:26:17,459 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 137 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:26:17,459 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083, journal: Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083, size=0 (0bytes) at 1733340373314Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000000001-wal.1733340373083.temp at 1733340377334 (+4020 ms)Split 1024 edits, skipped 0 edits. at 1733340377398 (+64 ms)Split 2048 edits, skipped 0 edits. at 1733340377428 (+30 ms)Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 so closing down at 1733340377451 (+23 ms)3 split writer threads finished at 1733340377451Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000000001-wal.1733340373083.temp (wrote 3002 edits, skipped 0 edits in 47 ms) at 1733340377457 (+6 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000000001-wal.1733340373083.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002 at 1733340377459 (+2 ms)Processed 3002 edits across 1 Regions in 137 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083, size=0, length=0, corrupted=false, cancelled=false at 1733340377459 2024-12-04T19:26:17,461 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340373083 2024-12-04T19:26:17,463 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002 2024-12-04T19:26:17,463 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:17,466 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:17,484 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340377466, exclude list is [], retry=0 2024-12-04T19:26:17,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:17,489 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:17,489 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:17,497 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340377466 2024-12-04T19:26:17,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:17,498 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:17,500 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:17,501 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ffbd393a28d04d387295a767ed41401 columnFamilyName a 2024-12-04T19:26:17,502 DEBUG [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:17,502 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(327): Store=3ffbd393a28d04d387295a767ed41401/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:17,503 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:17,504 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ffbd393a28d04d387295a767ed41401 columnFamilyName b 2024-12-04T19:26:17,504 DEBUG [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:17,504 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(327): Store=3ffbd393a28d04d387295a767ed41401/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:17,505 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:17,506 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ffbd393a28d04d387295a767ed41401 columnFamilyName c 2024-12-04T19:26:17,506 DEBUG [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:17,507 INFO [StoreOpener-3ffbd393a28d04d387295a767ed41401-1 {}] regionserver.HStore(327): Store=3ffbd393a28d04d387295a767ed41401/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:17,507 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:17,509 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:17,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:17,512 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002 2024-12-04T19:26:17,515 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:17,565 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:26:17,972 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ffbd393a28d04d387295a767ed41401 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T19:26:18,022 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/a/936c138934dc4605bbcee8cd0b4b76a7 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733340373113/Put/seqid=0 2024-12-04T19:26:18,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741858_1035 (size=50463) 2024-12-04T19:26:18,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741858_1035 (size=50463) 2024-12-04T19:26:18,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741858_1035 (size=50463) 2024-12-04T19:26:18,041 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/a/936c138934dc4605bbcee8cd0b4b76a7 2024-12-04T19:26:18,052 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/a/936c138934dc4605bbcee8cd0b4b76a7 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/a/936c138934dc4605bbcee8cd0b4b76a7 2024-12-04T19:26:18,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/a/936c138934dc4605bbcee8cd0b4b76a7, entries=754, sequenceid=754, filesize=49.3 K 2024-12-04T19:26:18,064 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 3ffbd393a28d04d387295a767ed41401 in 92ms, sequenceid=754, compaction requested=false; wal=null 2024-12-04T19:26:18,093 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:26:18,093 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ffbd393a28d04d387295a767ed41401 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T19:26:18,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/a/84ae5edeb4584333970d58f98050b0e3 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733340373167/Put/seqid=0 2024-12-04T19:26:18,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741859_1036 (size=20072) 2024-12-04T19:26:18,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741859_1036 (size=20072) 2024-12-04T19:26:18,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741859_1036 (size=20072) 2024-12-04T19:26:18,121 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/a/84ae5edeb4584333970d58f98050b0e3 2024-12-04T19:26:18,152 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/b/fa28e5f3b80b4091a83380a76dbb3694 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733340373187/Put/seqid=0 2024-12-04T19:26:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741860_1037 (size=35835) 2024-12-04T19:26:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741860_1037 (size=35835) 2024-12-04T19:26:18,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741860_1037 (size=35835) 2024-12-04T19:26:18,161 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/b/fa28e5f3b80b4091a83380a76dbb3694 2024-12-04T19:26:18,170 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/a/84ae5edeb4584333970d58f98050b0e3 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/a/84ae5edeb4584333970d58f98050b0e3 2024-12-04T19:26:18,180 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/a/84ae5edeb4584333970d58f98050b0e3, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-04T19:26:18,182 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/b/fa28e5f3b80b4091a83380a76dbb3694 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/b/fa28e5f3b80b4091a83380a76dbb3694 2024-12-04T19:26:18,193 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/b/fa28e5f3b80b4091a83380a76dbb3694, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-04T19:26:18,193 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 3ffbd393a28d04d387295a767ed41401 in 100ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-04T19:26:18,210 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:26:18,211 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ffbd393a28d04d387295a767ed41401 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T19:26:18,219 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/b/e0f04ab594f14ae8bcacb64946927e67 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733340373208/Put/seqid=0 2024-12-04T19:26:18,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741861_1038 (size=35082) 2024-12-04T19:26:18,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741861_1038 (size=35082) 2024-12-04T19:26:18,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741861_1038 (size=35082) 2024-12-04T19:26:18,228 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/b/e0f04ab594f14ae8bcacb64946927e67 2024-12-04T19:26:18,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/c/f09c19acc50345ad81e8063c2f83be4c is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733340373238/Put/seqid=0 2024-12-04T19:26:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741862_1039 (size=20825) 2024-12-04T19:26:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741862_1039 (size=20825) 2024-12-04T19:26:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741862_1039 (size=20825) 2024-12-04T19:26:18,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/c/f09c19acc50345ad81e8063c2f83be4c 2024-12-04T19:26:18,281 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/b/e0f04ab594f14ae8bcacb64946927e67 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/b/e0f04ab594f14ae8bcacb64946927e67 2024-12-04T19:26:18,290 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/b/e0f04ab594f14ae8bcacb64946927e67, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-04T19:26:18,293 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/c/f09c19acc50345ad81e8063c2f83be4c as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/c/f09c19acc50345ad81e8063c2f83be4c 2024-12-04T19:26:18,302 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/c/f09c19acc50345ad81e8063c2f83be4c, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-04T19:26:18,303 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 3ffbd393a28d04d387295a767ed41401 in 92ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-04T19:26:18,316 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733340373284/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:18,323 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002 2024-12-04T19:26:18,323 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:26:18,324 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ffbd393a28d04d387295a767ed41401 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-04T19:26:18,342 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/c/7985a4ed508546f8bfcd225be94d4ca3 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733340373247/Put/seqid=0 2024-12-04T19:26:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741863_1040 (size=50301) 2024-12-04T19:26:18,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741863_1040 (size=50301) 2024-12-04T19:26:18,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741863_1040 (size=50301) 2024-12-04T19:26:18,360 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/c/7985a4ed508546f8bfcd225be94d4ca3 2024-12-04T19:26:18,368 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7985a4ed508546f8bfcd225be94d4ca3 2024-12-04T19:26:18,370 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/.tmp/c/7985a4ed508546f8bfcd225be94d4ca3 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/c/7985a4ed508546f8bfcd225be94d4ca3 2024-12-04T19:26:18,415 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7985a4ed508546f8bfcd225be94d4ca3 2024-12-04T19:26:18,415 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/c/7985a4ed508546f8bfcd225be94d4ca3, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-04T19:26:18,416 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 3ffbd393a28d04d387295a767ed41401 in 92ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-04T19:26:18,420 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/0000000000000003002 2024-12-04T19:26:18,421 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:18,421 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:18,422 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:26:18,425 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ffbd393a28d04d387295a767ed41401 2024-12-04T19:26:18,428 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenIntoWAL/3ffbd393a28d04d387295a767ed41401/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-04T19:26:18,432 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ffbd393a28d04d387295a767ed41401; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66397516, jitterRate=-0.010599911212921143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:26:18,432 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ffbd393a28d04d387295a767ed41401: Writing region info on filesystem at 1733340377498Initializing all the Stores at 1733340377499 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340377499Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340377499Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340377500 (+1 ms)Cleaning up temporary data from old regions at 1733340378421 (+921 ms)Region opened successfully at 1733340378432 (+11 ms) 2024-12-04T19:26:18,518 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ffbd393a28d04d387295a767ed41401, disabling compactions & flushes 2024-12-04T19:26:18,518 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:18,518 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:18,519 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. after waiting 0 ms 2024-12-04T19:26:18,519 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:18,529 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340373022.3ffbd393a28d04d387295a767ed41401. 2024-12-04T19:26:18,529 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ffbd393a28d04d387295a767ed41401: Waiting for close lock at 1733340378518Disabling compacts and flushes for region at 1733340378518Disabling writes for close at 1733340378519 (+1 ms)Writing region close event to WAL at 1733340378529 (+10 ms)Closed at 1733340378529 2024-12-04T19:26:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741857_1034 (size=93) 2024-12-04T19:26:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741857_1034 (size=93) 2024-12-04T19:26:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741857_1034 (size=93) 2024-12-04T19:26:18,539 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:18,539 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340377466) 2024-12-04T19:26:18,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=402 (was 388) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_701499106_22 at /127.0.0.1:43734 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_701499106_22 at /127.0.0.1:55758 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:43251 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33061 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44731 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43251 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33061 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_701499106_22 at /127.0.0.1:54194 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_701499106_22 at /127.0.0.1:54216 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 235) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=16291 (was 16238) - AvailableMemoryMB LEAK? - 2024-12-04T19:26:18,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=16291 2024-12-04T19:26:18,599 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:18,602 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:18,603 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:18,606 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-96072536, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-96072536, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:18,625 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-96072536/hregion-96072536.1733340378607, exclude list is [], retry=0 2024-12-04T19:26:18,628 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:18,628 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:18,631 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:18,879 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-96072536/hregion-96072536.1733340378607 2024-12-04T19:26:18,880 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:18,880 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9f653d1c4a29606de807f8760b04cec2, NAME => 'test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741865_1042 (size=43) 2024-12-04T19:26:18,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741865_1042 (size=43) 2024-12-04T19:26:18,905 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:18,906 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,908 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f653d1c4a29606de807f8760b04cec2 columnFamilyName a 2024-12-04T19:26:18,908 DEBUG [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:18,908 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(327): Store=9f653d1c4a29606de807f8760b04cec2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:18,909 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741865_1042 (size=43) 2024-12-04T19:26:18,910 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f653d1c4a29606de807f8760b04cec2 columnFamilyName b 2024-12-04T19:26:18,910 DEBUG [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:18,911 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(327): Store=9f653d1c4a29606de807f8760b04cec2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:18,911 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,912 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f653d1c4a29606de807f8760b04cec2 columnFamilyName c 2024-12-04T19:26:18,912 DEBUG [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:18,913 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(327): Store=9f653d1c4a29606de807f8760b04cec2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:18,913 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,914 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,914 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,916 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,916 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,916 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:18,918 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:18,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T19:26:18,919 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:18,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:26:18,920 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T19:26:18,920 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:18,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T19:26:18,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:18,921 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9f653d1c4a29606de807f8760b04cec2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65328992, jitterRate=-0.026522159576416016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:18,921 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-04T19:26:18,921 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:18,922 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9f653d1c4a29606de807f8760b04cec2: Writing region info on filesystem at 1733340378905Initializing all the Stores at 1733340378906 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340378906Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340378906Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340378906Cleaning up temporary data from old regions at 1733340378916 (+10 ms)Region opened successfully at 1733340378922 (+6 ms) 2024-12-04T19:26:18,922 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9f653d1c4a29606de807f8760b04cec2, disabling compactions & flushes 2024-12-04T19:26:18,923 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:18,923 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:18,923 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. after waiting 0 ms 2024-12-04T19:26:18,923 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:18,923 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:18,923 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9f653d1c4a29606de807f8760b04cec2: Waiting for close lock at 1733340378922Disabling compacts and flushes for region at 1733340378922Disabling writes for close at 1733340378923 (+1 ms)Writing region close event to WAL at 1733340378923Closed at 1733340378923 2024-12-04T19:26:18,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741864_1041 (size=93) 2024-12-04T19:26:18,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741864_1041 (size=93) 2024-12-04T19:26:18,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741864_1041 (size=93) 2024-12-04T19:26:18,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:18,929 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-96072536:(num 1733340378607) 2024-12-04T19:26:18,929 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:18,933 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:18,948 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933, exclude list is [], retry=0 2024-12-04T19:26:18,950 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:18,951 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:18,951 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:18,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 2024-12-04T19:26:18,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741866_1043 (size=200357) 2024-12-04T19:26:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741866_1043 (size=200357) 2024-12-04T19:26:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741866_1043 (size=200357) 2024-12-04T19:26:19,147 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933, size=195.7 K (200357bytes) 2024-12-04T19:26:19,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 2024-12-04T19:26:19,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 after 0ms 2024-12-04T19:26:19,150 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:19,152 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 took 6ms 2024-12-04T19:26:19,156 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733340378933.temp 2024-12-04T19:26:19,157 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000000001-wal.1733340378933.temp 2024-12-04T19:26:19,204 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 so closing down 2024-12-04T19:26:19,204 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:19,204 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:19,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741867_1044 (size=200357) 2024-12-04T19:26:19,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741867_1044 (size=200357) 2024-12-04T19:26:19,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741867_1044 (size=200357) 2024-12-04T19:26:19,216 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000000001-wal.1733340378933.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-12-04T19:26:19,217 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000000001-wal.1733340378933.temp to hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000 2024-12-04T19:26:19,218 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-04T19:26:19,218 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933, journal: Splitting hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933, size=195.7 K (200357bytes) at 1733340379147Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000000001-wal.1733340378933.temp at 1733340379157 (+10 ms)Split 1024 edits, skipped 0 edits. at 1733340379170 (+13 ms)Split 2048 edits, skipped 0 edits. at 1733340379187 (+17 ms)Finishing writing output for hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 so closing down at 1733340379204 (+17 ms)3 split writer threads finished at 1733340379204Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000000001-wal.1733340378933.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1733340379216 (+12 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000000001-wal.1733340378933.temp to hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000 at 1733340379217 (+1 ms)Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733340379218 (+1 ms) 2024-12-04T19:26:19,219 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340378933 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340378933 2024-12-04T19:26:19,221 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000 2024-12-04T19:26:19,221 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:19,223 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:19,236 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224, exclude list is [], retry=0 2024-12-04T19:26:19,239 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:19,240 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:19,240 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:19,242 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 2024-12-04T19:26:19,242 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:19,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741868_1045 (size=200484) 2024-12-04T19:26:19,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741868_1045 (size=200484) 2024-12-04T19:26:19,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741868_1045 (size=200484) 2024-12-04T19:26:19,380 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224, size=195.8 K (200484bytes) 2024-12-04T19:26:19,381 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 2024-12-04T19:26:19,381 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 after 0ms 2024-12-04T19:26:19,384 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:19,386 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 took 6ms 2024-12-04T19:26:19,390 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733340379224.temp 2024-12-04T19:26:19,396 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003001-wal.1733340379224.temp 2024-12-04T19:26:19,432 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 so closing down 2024-12-04T19:26:19,432 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:19,434 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741869_1046 (size=200484) 2024-12-04T19:26:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741869_1046 (size=200484) 2024-12-04T19:26:19,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741869_1046 (size=200484) 2024-12-04T19:26:19,437 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003001-wal.1733340379224.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-12-04T19:26:19,439 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003001-wal.1733340379224.temp to hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000 2024-12-04T19:26:19,439 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-04T19:26:19,439 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224, journal: Splitting hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224, size=195.8 K (200484bytes) at 1733340379381Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003001-wal.1733340379224.temp at 1733340379396 (+15 ms)Split 1024 edits, skipped 0 edits. at 1733340379403 (+7 ms)Split 2048 edits, skipped 0 edits. at 1733340379418 (+15 ms)Finishing writing output for hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 so closing down at 1733340379432 (+14 ms)3 split writer threads finished at 1733340379434 (+2 ms)Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003001-wal.1733340379224.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1733340379437 (+3 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003001-wal.1733340379224.temp to hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000 at 1733340379439 (+2 ms)Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733340379439 2024-12-04T19:26:19,442 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379224 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340379224 2024-12-04T19:26:19,443 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000 2024-12-04T19:26:19,443 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:19,446 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/test2727-manual,16010,1733340378598, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:19,466 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379446, exclude list is [], retry=0 2024-12-04T19:26:19,469 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:19,470 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:19,470 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:19,472 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733340378598/wal.1733340379446 2024-12-04T19:26:19,472 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:19,472 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f653d1c4a29606de807f8760b04cec2, NAME => 'test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:19,472 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:19,472 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,472 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,474 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,475 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f653d1c4a29606de807f8760b04cec2 columnFamilyName a 2024-12-04T19:26:19,475 DEBUG [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:19,475 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(327): Store=9f653d1c4a29606de807f8760b04cec2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:19,476 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,477 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f653d1c4a29606de807f8760b04cec2 columnFamilyName b 2024-12-04T19:26:19,477 DEBUG [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:19,477 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(327): Store=9f653d1c4a29606de807f8760b04cec2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:19,477 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,478 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f653d1c4a29606de807f8760b04cec2 columnFamilyName c 2024-12-04T19:26:19,478 DEBUG [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:19,479 INFO [StoreOpener-9f653d1c4a29606de807f8760b04cec2-1 {}] regionserver.HStore(327): Store=9f653d1c4a29606de807f8760b04cec2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,482 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,483 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000 2024-12-04T19:26:19,485 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:19,526 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000 2024-12-04T19:26:19,527 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000 2024-12-04T19:26:19,530 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:19,565 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000 2024-12-04T19:26:19,566 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9f653d1c4a29606de807f8760b04cec2 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-04T19:26:19,586 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/a/21973913183d4d71b2e976d2cb90b918 is 41, key is test2727/a:100/1733340379246/Put/seqid=0 2024-12-04T19:26:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741871_1048 (size=84227) 2024-12-04T19:26:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741871_1048 (size=84227) 2024-12-04T19:26:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741871_1048 (size=84227) 2024-12-04T19:26:19,595 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/a/21973913183d4d71b2e976d2cb90b918 2024-12-04T19:26:19,622 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/b/5666558446e74612a0a20755dce32420 is 41, key is test2727/b:100/1733340379287/Put/seqid=0 2024-12-04T19:26:19,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741872_1049 (size=84609) 2024-12-04T19:26:19,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741872_1049 (size=84609) 2024-12-04T19:26:19,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741872_1049 (size=84609) 2024-12-04T19:26:19,652 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/b/5666558446e74612a0a20755dce32420 2024-12-04T19:26:19,679 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/c/16a20651763c47bca2ca05736dc7c663 is 41, key is test2727/c:100/1733340379325/Put/seqid=0 2024-12-04T19:26:19,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741873_1050 (size=84609) 2024-12-04T19:26:19,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741873_1050 (size=84609) 2024-12-04T19:26:19,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741873_1050 (size=84609) 2024-12-04T19:26:19,690 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/c/16a20651763c47bca2ca05736dc7c663 2024-12-04T19:26:19,699 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/a/21973913183d4d71b2e976d2cb90b918 as hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/a/21973913183d4d71b2e976d2cb90b918 2024-12-04T19:26:19,707 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/a/21973913183d4d71b2e976d2cb90b918, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-04T19:26:19,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/b/5666558446e74612a0a20755dce32420 as hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/b/5666558446e74612a0a20755dce32420 2024-12-04T19:26:19,716 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/b/5666558446e74612a0a20755dce32420, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T19:26:19,718 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/.tmp/c/16a20651763c47bca2ca05736dc7c663 as hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/c/16a20651763c47bca2ca05736dc7c663 2024-12-04T19:26:19,725 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/c/16a20651763c47bca2ca05736dc7c663, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T19:26:19,726 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 9f653d1c4a29606de807f8760b04cec2 in 161ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-04T19:26:19,727 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000003000 2024-12-04T19:26:19,727 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/0000000000000006000 2024-12-04T19:26:19,729 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,729 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,730 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:19,732 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9f653d1c4a29606de807f8760b04cec2 2024-12-04T19:26:19,739 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/test2727/9f653d1c4a29606de807f8760b04cec2/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-04T19:26:19,741 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9f653d1c4a29606de807f8760b04cec2; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62171089, jitterRate=-0.07357858121395111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:19,743 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9f653d1c4a29606de807f8760b04cec2: Writing region info on filesystem at 1733340379473Initializing all the Stores at 1733340379473Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340379473Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340379474 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340379474Obtaining lock to block concurrent updates at 1733340379566 (+92 ms)Preparing flush snapshotting stores in 9f653d1c4a29606de807f8760b04cec2 at 1733340379566Finished memstore snapshotting test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733340379566Flushing stores of test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. at 1733340379566Flushing 9f653d1c4a29606de807f8760b04cec2/a: creating writer at 1733340379566Flushing 9f653d1c4a29606de807f8760b04cec2/a: appending metadata at 1733340379585 (+19 ms)Flushing 9f653d1c4a29606de807f8760b04cec2/a: closing flushed file at 1733340379585Flushing 9f653d1c4a29606de807f8760b04cec2/b: creating writer at 1733340379601 (+16 ms)Flushing 9f653d1c4a29606de807f8760b04cec2/b: appending metadata at 1733340379620 (+19 ms)Flushing 9f653d1c4a29606de807f8760b04cec2/b: closing flushed file at 1733340379620Flushing 9f653d1c4a29606de807f8760b04cec2/c: creating writer at 1733340379658 (+38 ms)Flushing 9f653d1c4a29606de807f8760b04cec2/c: appending metadata at 1733340379678 (+20 ms)Flushing 9f653d1c4a29606de807f8760b04cec2/c: closing flushed file at 1733340379678Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6807576: reopening flushed file at 1733340379698 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d94ebfa: reopening flushed file at 1733340379707 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48062a04: reopening flushed file at 1733340379717 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 9f653d1c4a29606de807f8760b04cec2 in 161ms, sequenceid=6000, compaction requested=false; wal=null at 1733340379726 (+9 ms)Cleaning up temporary data from old regions at 1733340379729 (+3 ms)Region opened successfully at 1733340379743 (+14 ms) 2024-12-04T19:26:19,745 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-04T19:26:19,745 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9f653d1c4a29606de807f8760b04cec2, disabling compactions & flushes 2024-12-04T19:26:19,745 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:19,745 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:19,745 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. after waiting 0 ms 2024-12-04T19:26:19,745 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:19,754 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733340378600.9f653d1c4a29606de807f8760b04cec2. 2024-12-04T19:26:19,754 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9f653d1c4a29606de807f8760b04cec2: Waiting for close lock at 1733340379745Disabling compacts and flushes for region at 1733340379745Disabling writes for close at 1733340379745Writing region close event to WAL at 1733340379754 (+9 ms)Closed at 1733340379754 2024-12-04T19:26:19,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741870_1047 (size=93) 2024-12-04T19:26:19,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741870_1047 (size=93) 2024-12-04T19:26:19,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741870_1047 (size=93) 2024-12-04T19:26:19,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:19,762 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340379446) 2024-12-04T19:26:19,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=407 (was 402) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:55876 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54194 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43832 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54216 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 248) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=16071 (was 16291) 2024-12-04T19:26:19,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=407, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=16065 2024-12-04T19:26:19,810 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:19,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:19,818 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733340379818 2024-12-04T19:26:19,838 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818 2024-12-04T19:26:19,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:19,841 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 66b1ca9bdad4d8990843038473f90d54, NAME => 'testSequentialEditLogSeqNum,,1733340379811.66b1ca9bdad4d8990843038473f90d54.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:19,842 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733340379811.66b1ca9bdad4d8990843038473f90d54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:19,842 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,842 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,843 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54 doesn't exist for region: 66b1ca9bdad4d8990843038473f90d54 on table testSequentialEditLogSeqNum 2024-12-04T19:26:19,843 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 66b1ca9bdad4d8990843038473f90d54 on table testSequentialEditLogSeqNum 2024-12-04T19:26:19,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741875_1052 (size=62) 2024-12-04T19:26:19,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741875_1052 (size=62) 2024-12-04T19:26:19,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741875_1052 (size=62) 2024-12-04T19:26:19,864 INFO [StoreOpener-66b1ca9bdad4d8990843038473f90d54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,866 INFO [StoreOpener-66b1ca9bdad4d8990843038473f90d54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66b1ca9bdad4d8990843038473f90d54 columnFamilyName a 2024-12-04T19:26:19,867 DEBUG [StoreOpener-66b1ca9bdad4d8990843038473f90d54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:19,867 INFO [StoreOpener-66b1ca9bdad4d8990843038473f90d54-1 {}] regionserver.HStore(327): Store=66b1ca9bdad4d8990843038473f90d54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:19,868 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,869 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,869 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 66b1ca9bdad4d8990843038473f90d54 2024-12-04T19:26:19,876 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:19,877 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 66b1ca9bdad4d8990843038473f90d54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62470924, jitterRate=-0.0691106915473938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:19,878 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 66b1ca9bdad4d8990843038473f90d54: Writing region info on filesystem at 1733340379842Initializing all the Stores at 1733340379861 (+19 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340379861Cleaning up temporary data from old regions at 1733340379870 (+9 ms)Region opened successfully at 1733340379878 (+8 ms) 2024-12-04T19:26:19,891 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 66b1ca9bdad4d8990843038473f90d54 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-04T19:26:19,917 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/.tmp/a/c85f5767407a443db28fba5ecb8a5fb1 is 81, key is testSequentialEditLogSeqNum/a:x0/1733340379878/Put/seqid=0 2024-12-04T19:26:19,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741876_1053 (size=5833) 2024-12-04T19:26:19,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741876_1053 (size=5833) 2024-12-04T19:26:19,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741876_1053 (size=5833) 2024-12-04T19:26:19,930 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/.tmp/a/c85f5767407a443db28fba5ecb8a5fb1 2024-12-04T19:26:19,937 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/.tmp/a/c85f5767407a443db28fba5ecb8a5fb1 as hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/a/c85f5767407a443db28fba5ecb8a5fb1 2024-12-04T19:26:19,946 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/a/c85f5767407a443db28fba5ecb8a5fb1, entries=10, sequenceid=13, filesize=5.7 K 2024-12-04T19:26:19,948 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 66b1ca9bdad4d8990843038473f90d54 in 57ms, sequenceid=13, compaction requested=false 2024-12-04T19:26:19,948 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 66b1ca9bdad4d8990843038473f90d54: 2024-12-04T19:26:19,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:26:19,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:26:19,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:26:19,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:26:19,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:26:19,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741874_1051 (size=1616) 2024-12-04T19:26:19,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741874_1051 (size=1616) 2024-12-04T19:26:19,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741874_1051 (size=1616) 2024-12-04T19:26:19,986 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818, size=1.6 K (1616bytes) 2024-12-04T19:26:19,986 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818 2024-12-04T19:26:19,987 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818 after 1ms 2024-12-04T19:26:19,992 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:19,993 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818 took 7ms 2024-12-04T19:26:19,998 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818 so closing down 2024-12-04T19:26:19,998 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:19,999 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340379818.temp 2024-12-04T19:26:20,004 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000003-wal.1733340379818.temp 2024-12-04T19:26:20,005 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:20,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741877_1054 (size=1175) 2024-12-04T19:26:20,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741877_1054 (size=1175) 2024-12-04T19:26:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741877_1054 (size=1175) 2024-12-04T19:26:20,022 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000003-wal.1733340379818.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:20,025 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000003-wal.1733340379818.temp to hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000020 2024-12-04T19:26:20,025 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 30 ms; skipped=2; WAL=hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-04T19:26:20,026 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818, journal: Splitting hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818, size=1.6 K (1616bytes) at 1733340379986Finishing writing output for hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818 so closing down at 1733340379998 (+12 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000003-wal.1733340379818.temp at 1733340380004 (+6 ms)3 split writer threads finished at 1733340380005 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000003-wal.1733340379818.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733340380022 (+17 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000003-wal.1733340379818.temp to hdfs://localhost:33061/hbase/data/default/testSequentialEditLogSeqNum/66b1ca9bdad4d8990843038473f90d54/recovered.edits/0000000000000000020 at 1733340380025 (+3 ms)Processed 17 edits across 1 Regions in 30 ms; skipped=2; WAL=hdfs://localhost:33061/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340379809/wal.1733340379818, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733340380025 2024-12-04T19:26:20,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=412 (was 407) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:55876 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43832 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 292), ProcessCount=11 (was 11), AvailableMemoryMB=16002 (was 16065) 2024-12-04T19:26:20,070 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=412, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=15997 2024-12-04T19:26:20,097 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:20,101 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:20,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:20,162 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66034861, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-66034861, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:20,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66034861/hregion-66034861.1733340380163, exclude list is [], retry=0 2024-12-04T19:26:20,201 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:20,214 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:20,219 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:20,240 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66034861/hregion-66034861.1733340380163 2024-12-04T19:26:20,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:20,248 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 858b1730afd75bef899c5d85dcfd18af, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:20,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741879_1056 (size=70) 2024-12-04T19:26:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741879_1056 (size=70) 2024-12-04T19:26:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741879_1056 (size=70) 2024-12-04T19:26:20,272 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:20,279 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,281 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName a 2024-12-04T19:26:20,281 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:20,281 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:20,282 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,284 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName b 2024-12-04T19:26:20,285 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:20,285 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:20,286 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,288 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName c 2024-12-04T19:26:20,288 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:20,289 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:20,290 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,291 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,291 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,293 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,293 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,294 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:20,297 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,301 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:20,302 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 858b1730afd75bef899c5d85dcfd18af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68360215, jitterRate=0.01864658296108246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:20,304 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 858b1730afd75bef899c5d85dcfd18af: Writing region info on filesystem at 1733340380272Initializing all the Stores at 1733340380273 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340380273Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340380278 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340380278Cleaning up temporary data from old regions at 1733340380293 (+15 ms)Region opened successfully at 1733340380304 (+11 ms) 2024-12-04T19:26:20,304 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 858b1730afd75bef899c5d85dcfd18af, disabling compactions & flushes 2024-12-04T19:26:20,304 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:20,304 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:20,304 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. after waiting 0 ms 2024-12-04T19:26:20,304 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:20,305 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:20,305 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 858b1730afd75bef899c5d85dcfd18af: Waiting for close lock at 1733340380304Disabling compacts and flushes for region at 1733340380304Disabling writes for close at 1733340380304Writing region close event to WAL at 1733340380305 (+1 ms)Closed at 1733340380305 2024-12-04T19:26:20,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741878_1055 (size=93) 2024-12-04T19:26:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741878_1055 (size=93) 2024-12-04T19:26:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741878_1055 (size=93) 2024-12-04T19:26:20,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:20,325 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66034861:(num 1733340380163) 2024-12-04T19:26:20,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:20,328 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:20,367 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329, exclude list is [], retry=0 2024-12-04T19:26:20,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:20,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:20,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:20,377 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 2024-12-04T19:26:20,379 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:20,380 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 858b1730afd75bef899c5d85dcfd18af, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:20,380 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:20,380 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,380 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,382 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,383 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName a 2024-12-04T19:26:20,383 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:20,384 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:20,384 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,385 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName b 2024-12-04T19:26:20,385 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:20,386 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:20,386 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,387 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName c 2024-12-04T19:26:20,388 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:20,388 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:20,389 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,392 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,393 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,393 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:20,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:20,396 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 858b1730afd75bef899c5d85dcfd18af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68475374, jitterRate=0.02036258578300476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:20,397 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 858b1730afd75bef899c5d85dcfd18af: Writing region info on filesystem at 1733340380380Initializing all the Stores at 1733340380381 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340380381Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340380382 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340380382Cleaning up temporary data from old regions at 1733340380393 (+11 ms)Region opened successfully at 1733340380397 (+4 ms) 2024-12-04T19:26:20,400 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733340380399/Put/seqid=0 2024-12-04T19:26:20,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741881_1058 (size=4826) 2024-12-04T19:26:20,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741881_1058 (size=4826) 2024-12-04T19:26:20,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741881_1058 (size=4826) 2024-12-04T19:26:20,411 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33061/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 858b1730afd75bef899c5d85dcfd18af/a 2024-12-04T19:26:20,421 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-04T19:26:20,421 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:26:20,421 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 858b1730afd75bef899c5d85dcfd18af: 2024-12-04T19:26:20,424 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_ 2024-12-04T19:26:20,425 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33061/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 858b1730afd75bef899c5d85dcfd18af/a as hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_ - updating store file list. 2024-12-04T19:26:20,432 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:20,432 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_ into 858b1730afd75bef899c5d85dcfd18af/a 2024-12-04T19:26:20,432 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33061/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 858b1730afd75bef899c5d85dcfd18af/a (new location: hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_) 2024-12-04T19:26:20,479 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329, size=0 (0bytes) 2024-12-04T19:26:20,479 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 might be still open, length is 0 2024-12-04T19:26:20,479 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 2024-12-04T19:26:20,480 WARN [IPC Server handler 1 on default port 33061 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-04T19:26:20,480 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 after 1ms 2024-12-04T19:26:22,647 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:55978 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55978 dst: /127.0.0.1:45577 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45577 remote=/127.0.0.1:55978]. Total timeout mills is 60000, 57793 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:22,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:43912 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43912 dst: /127.0.0.1:42863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:22,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54388 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41073:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54388 dst: /127.0.0.1:41073 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:22,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741880_1059 (size=434) 2024-12-04T19:26:22,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741880_1059 (size=434) 2024-12-04T19:26:22,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741880_1059 (size=434) 2024-12-04T19:26:24,481 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 after 4002ms 2024-12-04T19:26:24,485 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:24,485 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 took 4006ms 2024-12-04T19:26:24,487 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329; continuing. 2024-12-04T19:26:24,487 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 so closing down 2024-12-04T19:26:24,487 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:24,489 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733340380329.temp 2024-12-04T19:26:24,490 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005-wal.1733340380329.temp 2024-12-04T19:26:24,491 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:24,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741882_1060 (size=236) 2024-12-04T19:26:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741882_1060 (size=236) 2024-12-04T19:26:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741882_1060 (size=236) 2024-12-04T19:26:24,497 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005-wal.1733340380329.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:24,499 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005-wal.1733340380329.temp to hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005 2024-12-04T19:26:24,499 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:26:24,499 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329, journal: Splitting hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329, size=0 (0bytes) at 1733340380479Finishing writing output for hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 so closing down at 1733340384487 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005-wal.1733340380329.temp at 1733340384490 (+3 ms)3 split writer threads finished at 1733340384491 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005-wal.1733340380329.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733340384497 (+6 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005-wal.1733340380329.temp to hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005 at 1733340384499 (+2 ms)Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329, size=0, length=0, corrupted=false, cancelled=false at 1733340384499 2024-12-04T19:26:24,500 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340380329 2024-12-04T19:26:24,501 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005 2024-12-04T19:26:24,501 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:24,503 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:24,515 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340384503, exclude list is [], retry=0 2024-12-04T19:26:24,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:24,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:24,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:24,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340384503 2024-12-04T19:26:24,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:24,521 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 858b1730afd75bef899c5d85dcfd18af, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:24,521 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:24,521 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,521 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,522 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,523 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName a 2024-12-04T19:26:24,523 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:24,529 DEBUG [StoreFileOpener-858b1730afd75bef899c5d85dcfd18af-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:24,529 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/a0a452da24e64508bb8bca7b2f1a851f_SeqId_3_ 2024-12-04T19:26:24,529 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:24,530 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,531 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName b 2024-12-04T19:26:24,531 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:24,531 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:24,531 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,532 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 858b1730afd75bef899c5d85dcfd18af columnFamilyName c 2024-12-04T19:26:24,532 DEBUG [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:24,533 INFO [StoreOpener-858b1730afd75bef899c5d85dcfd18af-1 {}] regionserver.HStore(327): Store=858b1730afd75bef899c5d85dcfd18af/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:24,533 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,534 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,536 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,537 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005 2024-12-04T19:26:24,540 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:24,541 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005 2024-12-04T19:26:24,541 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 858b1730afd75bef899c5d85dcfd18af 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-04T19:26:24,556 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/.tmp/a/60c634b3874d496f802176bf33a89750 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733340380438/Put/seqid=0 2024-12-04T19:26:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741884_1062 (size=5149) 2024-12-04T19:26:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741884_1062 (size=5149) 2024-12-04T19:26:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741884_1062 (size=5149) 2024-12-04T19:26:24,563 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/.tmp/a/60c634b3874d496f802176bf33a89750 2024-12-04T19:26:24,569 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/.tmp/a/60c634b3874d496f802176bf33a89750 as hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/60c634b3874d496f802176bf33a89750 2024-12-04T19:26:24,575 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/a/60c634b3874d496f802176bf33a89750, entries=1, sequenceid=5, filesize=5.0 K 2024-12-04T19:26:24,576 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 858b1730afd75bef899c5d85dcfd18af in 35ms, sequenceid=5, compaction requested=false; wal=null 2024-12-04T19:26:24,577 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/0000000000000000005 2024-12-04T19:26:24,578 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,578 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,579 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:24,581 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 858b1730afd75bef899c5d85dcfd18af 2024-12-04T19:26:24,583 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/858b1730afd75bef899c5d85dcfd18af/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T19:26:24,584 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 858b1730afd75bef899c5d85dcfd18af; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72684363, jitterRate=0.08308140933513641}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:24,584 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 858b1730afd75bef899c5d85dcfd18af: Writing region info on filesystem at 1733340384521Initializing all the Stores at 1733340384522 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340384522Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340384522Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340384522Obtaining lock to block concurrent updates at 1733340384541 (+19 ms)Preparing flush snapshotting stores in 858b1730afd75bef899c5d85dcfd18af at 1733340384541Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733340384541Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. at 1733340384541Flushing 858b1730afd75bef899c5d85dcfd18af/a: creating writer at 1733340384541Flushing 858b1730afd75bef899c5d85dcfd18af/a: appending metadata at 1733340384555 (+14 ms)Flushing 858b1730afd75bef899c5d85dcfd18af/a: closing flushed file at 1733340384555Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3af83486: reopening flushed file at 1733340384568 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 858b1730afd75bef899c5d85dcfd18af in 35ms, sequenceid=5, compaction requested=false; wal=null at 1733340384576 (+8 ms)Cleaning up temporary data from old regions at 1733340384578 (+2 ms)Region opened successfully at 1733340384584 (+6 ms) 2024-12-04T19:26:24,588 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 858b1730afd75bef899c5d85dcfd18af, disabling compactions & flushes 2024-12-04T19:26:24,588 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:24,588 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:24,588 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. after waiting 0 ms 2024-12-04T19:26:24,588 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:24,589 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733340380098.858b1730afd75bef899c5d85dcfd18af. 2024-12-04T19:26:24,589 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 858b1730afd75bef899c5d85dcfd18af: Waiting for close lock at 1733340384588Disabling compacts and flushes for region at 1733340384588Disabling writes for close at 1733340384588Writing region close event to WAL at 1733340384588Closed at 1733340384588 2024-12-04T19:26:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741883_1061 (size=93) 2024-12-04T19:26:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741883_1061 (size=93) 2024-12-04T19:26:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741883_1061 (size=93) 2024-12-04T19:26:24,593 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:24,593 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340384503) 2024-12-04T19:26:24,605 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=417 (was 412) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1564026114_22 at /127.0.0.1:60842 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33061 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1564026114_22 at /127.0.0.1:53614 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33061 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1564026114_22 at /127.0.0.1:36496 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 292), ProcessCount=11 (was 11), AvailableMemoryMB=15703 (was 15997) 2024-12-04T19:26:24,607 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-04T19:26:24,619 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=417, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=15702 2024-12-04T19:26:24,619 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-04T19:26:24,634 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:24,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:26:24,642 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:24,644 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7266e60d 2024-12-04T19:26:24,645 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:26:24,646 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:26:24,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:26:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T19:26:24,656 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:26:24,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-04T19:26:24,659 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:24,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:26:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:26:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741885_1063 (size=694) 2024-12-04T19:26:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741885_1063 (size=694) 2024-12-04T19:26:24,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741885_1063 (size=694) 2024-12-04T19:26:24,673 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90 2024-12-04T19:26:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741886_1064 (size=77) 2024-12-04T19:26:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741886_1064 (size=77) 2024-12-04T19:26:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741886_1064 (size=77) 2024-12-04T19:26:24,681 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:24,681 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:24,681 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:24,681 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:24,681 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:24,681 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:24,682 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:24,682 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340384681Disabling compacts and flushes for region at 1733340384681Disabling writes for close at 1733340384681Writing region close event to WAL at 1733340384682 (+1 ms)Closed at 1733340384682 2024-12-04T19:26:24,683 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:26:24,687 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733340384683"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733340384683"}]},"ts":"1733340384683"} 2024-12-04T19:26:24,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:26:24,691 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:26:24,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733340384691"}]},"ts":"1733340384691"} 2024-12-04T19:26:24,697 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-04T19:26:24,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:26:24,698 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:26:24,698 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:26:24,698 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T19:26:24,698 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:26:24,698 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:26:24,698 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T19:26:24,699 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:26:24,699 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:26:24,699 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T19:26:24,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:26:24,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN}] 2024-12-04T19:26:24,702 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN 2024-12-04T19:26:24,703 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN; state=OFFLINE, location=9ba2734cd39f,46131,1733340369309; forceNewPlan=false, retain=false 2024-12-04T19:26:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:26:24,858 INFO [9ba2734cd39f:42915 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T19:26:24,860 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPENING, regionLocation=9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:24,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN because future has completed 2024-12-04T19:26:24,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309}] 2024-12-04T19:26:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:26:25,019 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:26:25,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44857, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:26:25,033 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,034 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:25,035 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,035 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:25,035 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,036 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,038 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,040 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf1 2024-12-04T19:26:25,040 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:25,040 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:25,041 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,042 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf2 2024-12-04T19:26:25,042 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:25,042 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:25,043 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,043 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,044 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,044 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,045 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,045 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:26:25,047 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,049 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:25,049 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ef32ee4d500fb92aaaa8c18d0bf47143; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62791299, jitterRate=-0.06433673202991486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:26:25,050 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,050 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ef32ee4d500fb92aaaa8c18d0bf47143: Running coprocessor pre-open hook at 1733340385036Writing region info on filesystem at 1733340385036Initializing all the Stores at 1733340385037 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340385037Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340385037Cleaning up temporary data from old regions at 1733340385045 (+8 ms)Running coprocessor post-open hooks at 1733340385050 (+5 ms)Region opened successfully at 1733340385050 2024-12-04T19:26:25,052 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., pid=6, masterSystemTime=1733340385019 2024-12-04T19:26:25,054 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,054 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:25,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309 because future has completed 2024-12-04T19:26:25,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T19:26:25,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309 in 194 msec 2024-12-04T19:26:25,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T19:26:25,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN in 362 msec 2024-12-04T19:26:25,066 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:26:25,067 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733340385067"}]},"ts":"1733340385067"} 2024-12-04T19:26:25,069 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-04T19:26:25,071 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:26:25,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 420 msec 2024-12-04T19:26:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:26:25,292 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-04T19:26:25,292 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-04T19:26:25,294 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T19:26:25,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-04T19:26:25,301 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T19:26:25,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-04T19:26:25,311 ERROR [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41753 {}] util.ResourceLeakDetector(337): LEAK: RefCnt.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information. Recent access records: Created at: org.apache.hadoop.hbase.nio.RefCnt.(RefCnt.java:59) org.apache.hadoop.hbase.nio.RefCnt.create(RefCnt.java:54) org.apache.hadoop.hbase.regionserver.MemStoreLABImpl.(MemStoreLABImpl.java:107) jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) org.apache.hadoop.hbase.util.ReflectionUtils.instantiate(ReflectionUtils.java:61) org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:49) org.apache.hadoop.hbase.regionserver.MemStoreLAB.newInstance(MemStoreLAB.java:116) org.apache.hadoop.hbase.regionserver.SegmentFactory.createMutableSegment(SegmentFactory.java:82) org.apache.hadoop.hbase.regionserver.AbstractMemStore.resetActive(AbstractMemStore.java:92) org.apache.hadoop.hbase.regionserver.DefaultMemStore.snapshot(DefaultMemStore.java:108) org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.prepare(HStore.java:1958) org.apache.hadoop.hbase.regionserver.HRegion.lambda$internalPrepareFlushCache$5(HRegion.java:2863) java.base/java.util.TreeMap.forEach(TreeMap.java:1282) org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2862) org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2735) org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testSequentialEditLogSeqNum(AbstractTestWALReplay.java:847) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base/java.lang.reflect.Method.invoke(Method.java:568) org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:26:25,313 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=2] 2024-12-04T19:26:25,314 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:26:25,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:26:25,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ef32ee4d500fb92aaaa8c18d0bf47143, source=9ba2734cd39f,46131,1733340369309, destination=9ba2734cd39f,41753,1733340369441, warming up region on 9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:25,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ef32ee4d500fb92aaaa8c18d0bf47143, source=9ba2734cd39f,46131,1733340369309, destination=9ba2734cd39f,41753,1733340369441, running balancer 2024-12-04T19:26:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE 2024-12-04T19:26:25,332 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE 2024-12-04T19:26:25,335 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=CLOSING, regionLocation=9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:25,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(7855): Warmup {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:25,336 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,337 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf1 2024-12-04T19:26:25,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE because future has completed 2024-12-04T19:26:25,337 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:25,338 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:26:25,338 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:25,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309}] 2024-12-04T19:26:25,338 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,340 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf2 2024-12-04T19:26:25,340 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:25,340 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:25,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340385341Disabling compacts and flushes for region at 1733340385341Disabling writes for close at 1733340385341Writing region close event to WAL at 1733340385342 (+1 ms)Closed at 1733340385342 2024-12-04T19:26:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T19:26:25,498 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,499 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:26:25,500 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:25,500 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,500 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,501 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:25,501 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,501 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing ef32ee4d500fb92aaaa8c18d0bf47143 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-04T19:26:25,518 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/7fe8c18783bb485494a3ef933ce0b935 is 35, key is r1/cf1:q/1733340385316/Put/seqid=0 2024-12-04T19:26:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741887_1065 (size=4783) 2024-12-04T19:26:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741887_1065 (size=4783) 2024-12-04T19:26:25,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741887_1065 (size=4783) 2024-12-04T19:26:25,525 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/7fe8c18783bb485494a3ef933ce0b935 2024-12-04T19:26:25,532 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/7fe8c18783bb485494a3ef933ce0b935 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935 2024-12-04T19:26:25,538 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935, entries=1, sequenceid=5, filesize=4.7 K 2024-12-04T19:26:25,540 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for ef32ee4d500fb92aaaa8c18d0bf47143 in 38ms, sequenceid=5, compaction requested=false 2024-12-04T19:26:25,540 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-04T19:26:25,544 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T19:26:25,547 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,547 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340385500Running coprocessor pre-close hooks at 1733340385500Disabling compacts and flushes for region at 1733340385500Disabling writes for close at 1733340385501 (+1 ms)Obtaining lock to block concurrent updates at 1733340385501Preparing flush snapshotting stores in ef32ee4d500fb92aaaa8c18d0bf47143 at 1733340385501Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733340385502 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. at 1733340385503 (+1 ms)Flushing ef32ee4d500fb92aaaa8c18d0bf47143/cf1: creating writer at 1733340385503Flushing ef32ee4d500fb92aaaa8c18d0bf47143/cf1: appending metadata at 1733340385518 (+15 ms)Flushing ef32ee4d500fb92aaaa8c18d0bf47143/cf1: closing flushed file at 1733340385518Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b83abb8: reopening flushed file at 1733340385531 (+13 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for ef32ee4d500fb92aaaa8c18d0bf47143 in 38ms, sequenceid=5, compaction requested=false at 1733340385540 (+9 ms)Writing region close event to WAL at 1733340385541 (+1 ms)Running coprocessor post-close hooks at 1733340385545 (+4 ms)Closed at 1733340385547 (+2 ms) 2024-12-04T19:26:25,547 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding ef32ee4d500fb92aaaa8c18d0bf47143 move to 9ba2734cd39f,41753,1733340369441 record at close sequenceid=5 2024-12-04T19:26:25,550 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,551 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=CLOSED 2024-12-04T19:26:25,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309 because future has completed 2024-12-04T19:26:25,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T19:26:25,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309 in 216 msec 2024-12-04T19:26:25,558 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,41753,1733340369441; forceNewPlan=false, retain=false 2024-12-04T19:26:25,709 INFO [9ba2734cd39f:42915 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T19:26:25,709 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPENING, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:25,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE because future has completed 2024-12-04T19:26:25,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441}] 2024-12-04T19:26:25,874 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,874 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:25,875 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,875 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:25,875 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,875 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,878 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,881 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf1 2024-12-04T19:26:25,881 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:25,891 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935 2024-12-04T19:26:25,891 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:25,891 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,892 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf2 2024-12-04T19:26:25,892 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:25,893 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:25,893 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,894 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,895 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,896 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,896 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,897 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:26:25,898 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,899 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ef32ee4d500fb92aaaa8c18d0bf47143; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71486880, jitterRate=0.06523752212524414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:26:25,899 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:25,900 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ef32ee4d500fb92aaaa8c18d0bf47143: Running coprocessor pre-open hook at 1733340385875Writing region info on filesystem at 1733340385875Initializing all the Stores at 1733340385878 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340385878Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340385878Cleaning up temporary data from old regions at 1733340385896 (+18 ms)Running coprocessor post-open hooks at 1733340385899 (+3 ms)Region opened successfully at 1733340385900 (+1 ms) 2024-12-04T19:26:25,901 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., pid=9, masterSystemTime=1733340385867 2024-12-04T19:26:25,903 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,903 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:25,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPEN, openSeqNum=9, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:25,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441 because future has completed 2024-12-04T19:26:25,910 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42915 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=9ba2734cd39f,41753,1733340369441, table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T19:26:25,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T19:26:25,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441 in 197 msec 2024-12-04T19:26:25,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE in 582 msec 2024-12-04T19:26:25,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:26:25,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:26:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:57574 deadline: 1733340445943, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=41753 startCode=1733340369441. As of locationSeqNum=5. 2024-12-04T19:26:25,950 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=41753 startCode=1733340369441. As of locationSeqNum=5. 2024-12-04T19:26:25,950 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=41753 startCode=1733340369441. As of locationSeqNum=5. 2024-12-04T19:26:25,951 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,41753,1733340369441, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=41753 startCode=1733340369441. As of locationSeqNum=5. 2024-12-04T19:26:26,070 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef32ee4d500fb92aaaa8c18d0bf47143 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-04T19:26:26,085 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/fa6c52c998524aa196a727bb9bf13705 is 29, key is r1/cf1:/1733340386061/DeleteFamily/seqid=0 2024-12-04T19:26:26,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741888_1066 (size=4906) 2024-12-04T19:26:26,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741888_1066 (size=4906) 2024-12-04T19:26:26,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741888_1066 (size=4906) 2024-12-04T19:26:26,092 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,098 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,111 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf2/73bb5be82abc4741aefd41fd42f8ba42 is 29, key is r1/cf2:/1733340386061/DeleteFamily/seqid=0 2024-12-04T19:26:26,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741889_1067 (size=4906) 2024-12-04T19:26:26,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741889_1067 (size=4906) 2024-12-04T19:26:26,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741889_1067 (size=4906) 2024-12-04T19:26:26,118 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf2/73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,125 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,126 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/fa6c52c998524aa196a727bb9bf13705 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,132 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,133 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T19:26:26,134 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf2/73bb5be82abc4741aefd41fd42f8ba42 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,140 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,140 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T19:26:26,141 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for ef32ee4d500fb92aaaa8c18d0bf47143 in 71ms, sequenceid=12, compaction requested=false 2024-12-04T19:26:26,141 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ef32ee4d500fb92aaaa8c18d0bf47143: 2024-12-04T19:26:26,144 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-04T19:26:26,145 DEBUG [Time-limited test {}] regionserver.HStore(1541): ef32ee4d500fb92aaaa8c18d0bf47143/cf1 is initiating major compaction (all files) 2024-12-04T19:26:26,146 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:26:26,146 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:26,146 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ef32ee4d500fb92aaaa8c18d0bf47143/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,147 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935, hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705] into tmpdir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp, totalSize=9.5 K 2024-12-04T19:26:26,148 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7fe8c18783bb485494a3ef933ce0b935, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733340385316 2024-12-04T19:26:26,149 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fa6c52c998524aa196a727bb9bf13705, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T19:26:26,161 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ef32ee4d500fb92aaaa8c18d0bf47143#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:26:26,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741890_1068 (size=4626) 2024-12-04T19:26:26,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741890_1068 (size=4626) 2024-12-04T19:26:26,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741890_1068 (size=4626) 2024-12-04T19:26:26,177 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf1/e54e1e6be5ea48a7800299e13bc589da as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/e54e1e6be5ea48a7800299e13bc589da 2024-12-04T19:26:26,190 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in ef32ee4d500fb92aaaa8c18d0bf47143/cf1 of ef32ee4d500fb92aaaa8c18d0bf47143 into e54e1e6be5ea48a7800299e13bc589da(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:26:26,190 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ef32ee4d500fb92aaaa8c18d0bf47143: 2024-12-04T19:26:26,190 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T19:26:26,190 DEBUG [Time-limited test {}] regionserver.HStore(1541): ef32ee4d500fb92aaaa8c18d0bf47143/cf2 is initiating major compaction (all files) 2024-12-04T19:26:26,190 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:26:26,190 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:26:26,190 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ef32ee4d500fb92aaaa8c18d0bf47143/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,190 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42] into tmpdir=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp, totalSize=4.8 K 2024-12-04T19:26:26,191 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 73bb5be82abc4741aefd41fd42f8ba42, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T19:26:26,197 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ef32ee4d500fb92aaaa8c18d0bf47143#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:26:26,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741891_1069 (size=4592) 2024-12-04T19:26:26,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741891_1069 (size=4592) 2024-12-04T19:26:26,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741891_1069 (size=4592) 2024-12-04T19:26:26,212 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/.tmp/cf2/14cf72c88dca40fd886f6379e4454e50 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/14cf72c88dca40fd886f6379e4454e50 2024-12-04T19:26:26,220 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in ef32ee4d500fb92aaaa8c18d0bf47143/cf2 of ef32ee4d500fb92aaaa8c18d0bf47143 into 14cf72c88dca40fd886f6379e4454e50(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:26:26,220 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ef32ee4d500fb92aaaa8c18d0bf47143: 2024-12-04T19:26:26,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ef32ee4d500fb92aaaa8c18d0bf47143, source=9ba2734cd39f,41753,1733340369441, destination=9ba2734cd39f,46131,1733340369309, warming up region on 9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:26,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ef32ee4d500fb92aaaa8c18d0bf47143, source=9ba2734cd39f,41753,1733340369441, destination=9ba2734cd39f,46131,1733340369309, running balancer 2024-12-04T19:26:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE 2024-12-04T19:26:26,228 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE 2024-12-04T19:26:26,229 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=CLOSING, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:26,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(7855): Warmup {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:26,231 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE because future has completed 2024-12-04T19:26:26,232 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:26:26,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441}] 2024-12-04T19:26:26,232 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf1 2024-12-04T19:26:26,233 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:26,240 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935 2024-12-04T19:26:26,246 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/e54e1e6be5ea48a7800299e13bc589da 2024-12-04T19:26:26,252 INFO [StoreFileOpener-ef32ee4d500fb92aaaa8c18d0bf47143-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,252 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,252 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:26,252 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,253 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf2 2024-12-04T19:26:26,253 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:26,260 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/14cf72c88dca40fd886f6379e4454e50 2024-12-04T19:26:26,266 INFO [StoreFileOpener-ef32ee4d500fb92aaaa8c18d0bf47143-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,266 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,266 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:26,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46131 {}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340386266Disabling compacts and flushes for region at 1733340386266Disabling writes for close at 1733340386267 (+1 ms)Writing region close event to WAL at 1733340386268 (+1 ms)Closed at 1733340386268 2024-12-04T19:26:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-04T19:26:26,387 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,387 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:26:26,387 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:26,388 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,388 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,388 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:26,388 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,388 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935, hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705] to archive 2024-12-04T19:26:26,393 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:26:26,399 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935 to hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/7fe8c18783bb485494a3ef933ce0b935 2024-12-04T19:26:26,401 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705 to hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/fa6c52c998524aa196a727bb9bf13705 2024-12-04T19:26:26,412 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42] to archive 2024-12-04T19:26:26,413 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:26:26,415 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42 to hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/73bb5be82abc4741aefd41fd42f8ba42 2024-12-04T19:26:26,419 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-04T19:26:26,420 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,420 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340386387Running coprocessor pre-close hooks at 1733340386387Disabling compacts and flushes for region at 1733340386387Disabling writes for close at 1733340386388 (+1 ms)Writing region close event to WAL at 1733340386416 (+28 ms)Running coprocessor post-close hooks at 1733340386420 (+4 ms)Closed at 1733340386420 2024-12-04T19:26:26,420 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding ef32ee4d500fb92aaaa8c18d0bf47143 move to 9ba2734cd39f,46131,1733340369309 record at close sequenceid=12 2024-12-04T19:26:26,423 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,423 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=CLOSED 2024-12-04T19:26:26,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441 because future has completed 2024-12-04T19:26:26,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T19:26:26,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441 in 195 msec 2024-12-04T19:26:26,432 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,46131,1733340369309; forceNewPlan=false, retain=false 2024-12-04T19:26:26,583 INFO [9ba2734cd39f:42915 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T19:26:26,584 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPENING, regionLocation=9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:26,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE because future has completed 2024-12-04T19:26:26,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309}] 2024-12-04T19:26:26,747 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,748 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:26,748 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,748 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:26,748 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,748 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,750 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,751 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf1 2024-12-04T19:26:26,752 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:26,760 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/e54e1e6be5ea48a7800299e13bc589da 2024-12-04T19:26:26,760 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:26,760 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,761 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf2 2024-12-04T19:26:26,761 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:26,767 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/14cf72c88dca40fd886f6379e4454e50 2024-12-04T19:26:26,767 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:26,767 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,768 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,769 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,770 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,770 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,771 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:26:26,773 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,774 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened ef32ee4d500fb92aaaa8c18d0bf47143; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73141706, jitterRate=0.08989635109901428}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:26:26,774 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,774 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for ef32ee4d500fb92aaaa8c18d0bf47143: Running coprocessor pre-open hook at 1733340386749Writing region info on filesystem at 1733340386749Initializing all the Stores at 1733340386750 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340386750Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340386750Cleaning up temporary data from old regions at 1733340386770 (+20 ms)Running coprocessor post-open hooks at 1733340386774 (+4 ms)Region opened successfully at 1733340386774 2024-12-04T19:26:26,775 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., pid=12, masterSystemTime=1733340386743 2024-12-04T19:26:26,778 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,778 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,779 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPEN, openSeqNum=18, regionLocation=9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:26,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309 because future has completed 2024-12-04T19:26:26,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-04T19:26:26,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,46131,1733340369309 in 194 msec 2024-12-04T19:26:26,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, REOPEN/MOVE in 560 msec 2024-12-04T19:26:26,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:26:26,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:26:26,834 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9ba2734cd39f,46131,1733340369309: testing ***** 2024-12-04T19:26:26,834 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-04T19:26:26,836 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-04T19:26:26,837 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-04T19:26:26,840 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-04T19:26:26,842 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-04T19:26:26,850 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 310534632 }, "NonHeapMemoryUsage": { "committed": 171573248, "init": 7667712, "max": -1, "used": 168834064 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9ba2734cd39f", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 8, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 7, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 8, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 4, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 7, "TotalCallTime_98th_percentile": 7, "TotalCallTime_99th_percentile": 7, "TotalCallTime_99.9th_percentile": 7, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9ba2734cd39f", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59927", "tag.serverName": "9ba2734cd39f,46131,1733340369309", "tag.clusterId": "5ce29467-e57e-4952-bea7-e9dc4b39f4e6", "tag.Context": "regionserver", "tag.Hostname": "9ba2734cd39f", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733340369309, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-04T19:26:26,854 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42915 {}] master.MasterRpcServices(700): 9ba2734cd39f,46131,1733340369309 reported a fatal error: ***** ABORTING region server 9ba2734cd39f,46131,1733340369309: testing ***** 2024-12-04T19:26:26,857 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,46131,1733340369309' ***** 2024-12-04T19:26:26,857 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-04T19:26:26,857 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:26:26,857 INFO [RS:0;9ba2734cd39f:46131 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-04T19:26:26,857 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:26:26,857 INFO [RS:0;9ba2734cd39f:46131 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-04T19:26:26,857 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(3091): Received CLOSE for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,858 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(956): aborting server 9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:26,858 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:26:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:55542 deadline: 1733340446858, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46131 startCode=1733340369309. As of locationSeqNum=12. 2024-12-04T19:26:26,858 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:26,858 INFO [RS:0;9ba2734cd39f:46131 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9ba2734cd39f:46131. 2024-12-04T19:26:26,858 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,858 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,858 DEBUG [RS:0;9ba2734cd39f:46131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:26:26,859 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:26,859 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,859 DEBUG [RS:0;9ba2734cd39f:46131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:26,859 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,41753,1733340369441, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,41753,1733340369441, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46131 startCode=1733340369309. As of locationSeqNum=12. 2024-12-04T19:26:26,859 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T19:26:26,859 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,41753,1733340369441, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46131 startCode=1733340369309. As of locationSeqNum=12. 2024-12-04T19:26:26,859 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1325): Online Regions={ef32ee4d500fb92aaaa8c18d0bf47143=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.} 2024-12-04T19:26:26,859 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,41753,1733340369441, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46131 startCode=1733340369309. As of locationSeqNum=12. 2024-12-04T19:26:26,860 DEBUG [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1351): Waiting on ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:26,860 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,860 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340386858Running coprocessor pre-close hooks at 1733340386858Disabling compacts and flushes for region at 1733340386858Disabling writes for close at 1733340386859 (+1 ms)Writing region close event to WAL at 1733340386860 (+1 ms)Running coprocessor post-close hooks at 1733340386860Closed at 1733340386860 2024-12-04T19:26:26,860 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:26,888 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:26:26,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46131,1733340369309 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:26,972 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46131,1733340369309 aborting 2024-12-04T19:26:26,972 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46131,1733340369309 aborting 2024-12-04T19:26:26,972 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=12 from cache 2024-12-04T19:26:27,060 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,46131,1733340369309; all regions closed. 2024-12-04T19:26:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741833_1009 (size=1677) 2024-12-04T19:26:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741833_1009 (size=1677) 2024-12-04T19:26:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741833_1009 (size=1677) 2024-12-04T19:26:27,063 DEBUG [RS:0;9ba2734cd39f:46131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:27,063 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T19:26:27,064 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:26:27,064 INFO [RS:0;9ba2734cd39f:46131 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46131 2024-12-04T19:26:27,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:26:27,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,46131,1733340369309 2024-12-04T19:26:27,131 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:26:27,142 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,46131,1733340369309] 2024-12-04T19:26:27,151 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,46131,1733340369309 already deleted, retry=false 2024-12-04T19:26:27,151 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9ba2734cd39f,46131,1733340369309 on 9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:27,156 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9ba2734cd39f,46131,1733340369309, splitWal=true, meta=false 2024-12-04T19:26:27,158 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 9ba2734cd39f,46131,1733340369309 (carryingMeta=false) 9ba2734cd39f,46131,1733340369309/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1d7231cc[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T19:26:27,158 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9ba2734cd39f,46131,1733340369309, splitWal=true, meta=false 2024-12-04T19:26:27,160 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 9ba2734cd39f,46131,1733340369309 had 1 regions 2024-12-04T19:26:27,161 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9ba2734cd39f,46131,1733340369309, splitWal=true, meta=false, isMeta: false 2024-12-04T19:26:27,162 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting 2024-12-04T19:26:27,163 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9ba2734cd39f,46131,1733340369309 WAL count=1, meta=false 2024-12-04T19:26:27,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9ba2734cd39f%2C46131%2C1733340369309.1733340371226}] 2024-12-04T19:26:27,169 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:27,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9ba2734cd39f%2C46131%2C1733340369309.1733340371226, worker=9ba2734cd39f,34937,1733340369505}] 2024-12-04T19:26:27,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18] 2024-12-04T19:26:27,184 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9ba2734cd39f:46131 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ba2734cd39f/172.17.0.2:46131 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18, error=java.net.ConnectException: Call to address=9ba2734cd39f:46131 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ba2734cd39f/172.17.0.2:46131 2024-12-04T19:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-04T19:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18 from cache 2024-12-04T19:26:27,185 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 9ba2734cd39f:46131 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ba2734cd39f/172.17.0.2:46131 2024-12-04T19:26:27,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:27,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46131-0x101a1cf074a0001, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:27,243 INFO [RS:0;9ba2734cd39f:46131 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:26:27,243 INFO [RS:0;9ba2734cd39f:46131 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,46131,1733340369309; zookeeper connection closed. 2024-12-04T19:26:27,244 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@278c4278 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@278c4278 2024-12-04T19:26:27,331 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:26:27,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36607, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:26:27,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34937 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-04T19:26:27,352 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226, size=1.6 K (1677bytes) 2024-12-04T19:26:27,352 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 2024-12-04T19:26:27,353 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 after 1ms 2024-12-04T19:26:27,355 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:27,355 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 took 3ms 2024-12-04T19:26:27,361 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for ef32ee4d500fb92aaaa8c18d0bf47143: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-04T19:26:27,361 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 so closing down 2024-12-04T19:26:27,361 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:27,361 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:27,362 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 5 ms; skipped=6; WAL=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-04T19:26:27,362 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226, journal: Splitting hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226, size=1.6 K (1677bytes) at 1733340387352Finishing writing output for hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 so closing down at 1733340387361 (+9 ms)3 split writer threads finished at 1733340387361Processed 6 edits across 0 Regions in 5 ms; skipped=6; WAL=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733340387362 (+1 ms) 2024-12-04T19:26:27,362 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 2024-12-04T19:26:27,363 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-04T19:26:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42915 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-04T19:26:27,369 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting/9ba2734cd39f%2C46131%2C1733340369309.1733340371226 to hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs 2024-12-04T19:26:27,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T19:26:27,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9ba2734cd39f%2C46131%2C1733340369309.1733340371226, worker=9ba2734cd39f,34937,1733340369505 in 198 msec 2024-12-04T19:26:27,373 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:27,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T19:26:27,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9ba2734cd39f%2C46131%2C1733340369309.1733340371226, worker=9ba2734cd39f,34937,1733340369505 in 209 msec 2024-12-04T19:26:27,377 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting dir is empty, no logs to split. 2024-12-04T19:26:27,377 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9ba2734cd39f,46131,1733340369309 WAL count=0, meta=false 2024-12-04T19:26:27,377 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9ba2734cd39f,46131,1733340369309 WAL splitting is done? wals=0, meta=false 2024-12-04T19:26:27,379 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 9ba2734cd39f,46131,1733340369309 failed, ignore...File hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/WALs/9ba2734cd39f,46131,1733340369309-splitting does not exist. 2024-12-04T19:26:27,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN}] 2024-12-04T19:26:27,383 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN 2024-12-04T19:26:27,384 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T19:26:27,495 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18] 2024-12-04T19:26:27,496 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ba2734cd39f:46131 this server is in the failed servers list 2024-12-04T19:26:27,497 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ba2734cd39f:46131 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46131 2024-12-04T19:26:27,497 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46131 2024-12-04T19:26:27,498 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,46131,1733340369309, seqNum=18 from cache 2024-12-04T19:26:27,535 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:26:27,535 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:26:27,535 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:26:27,535 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:26:27,535 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:26:27,535 INFO [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:26:27,536 INFO [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:26:27,536 DEBUG [9ba2734cd39f:42915 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:26:27,536 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPENING, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:27,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN because future has completed 2024-12-04T19:26:27,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441}] 2024-12-04T19:26:27,701 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:27,701 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => ef32ee4d500fb92aaaa8c18d0bf47143, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:27,702 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,702 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:27,702 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,703 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,706 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,708 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf1 2024-12-04T19:26:27,708 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:27,713 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf1/e54e1e6be5ea48a7800299e13bc589da 2024-12-04T19:26:27,713 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:27,714 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,714 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef32ee4d500fb92aaaa8c18d0bf47143 columnFamilyName cf2 2024-12-04T19:26:27,714 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:27,720 DEBUG [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/cf2/14cf72c88dca40fd886f6379e4454e50 2024-12-04T19:26:27,720 INFO [StoreOpener-ef32ee4d500fb92aaaa8c18d0bf47143-1 {}] regionserver.HStore(327): Store=ef32ee4d500fb92aaaa8c18d0bf47143/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:27,721 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,721 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,722 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,723 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,723 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,723 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:26:27,725 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,725 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened ef32ee4d500fb92aaaa8c18d0bf47143; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69511026, jitterRate=0.035795003175735474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:26:27,725 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:27,726 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for ef32ee4d500fb92aaaa8c18d0bf47143: Running coprocessor pre-open hook at 1733340387703Writing region info on filesystem at 1733340387703Initializing all the Stores at 1733340387705 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340387705Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340387705Cleaning up temporary data from old regions at 1733340387723 (+18 ms)Running coprocessor post-open hooks at 1733340387725 (+2 ms)Region opened successfully at 1733340387726 (+1 ms) 2024-12-04T19:26:27,727 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., pid=17, masterSystemTime=1733340387693 2024-12-04T19:26:27,729 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:27,729 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:27,730 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ef32ee4d500fb92aaaa8c18d0bf47143, regionState=OPEN, openSeqNum=18, regionLocation=9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:27,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441 because future has completed 2024-12-04T19:26:27,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T19:26:27,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure ef32ee4d500fb92aaaa8c18d0bf47143, server=9ba2734cd39f,41753,1733340369441 in 193 msec 2024-12-04T19:26:27,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-04T19:26:27,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ef32ee4d500fb92aaaa8c18d0bf47143, ASSIGN in 353 msec 2024-12-04T19:26:27,738 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 9ba2734cd39f,46131,1733340369309 after splitting done 2024-12-04T19:26:27,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 9ba2734cd39f,46131,1733340369309, splitWal=true, meta=false in 586 msec 2024-12-04T19:26:28,007 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., hostname=9ba2734cd39f,41753,1733340369441, seqNum=18] 2024-12-04T19:26:28,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=412 (was 417), OpenFileDescriptor=1051 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 277), ProcessCount=11 (was 11), AvailableMemoryMB=15676 (was 15702) 2024-12-04T19:26:28,020 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-04T19:26:28,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=412, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=15675 2024-12-04T19:26:28,030 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-04T19:26:28,042 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:28,044 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:28,045 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:28,048 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43389463, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-43389463, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:28,059 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43389463/hregion-43389463.1733340388048, exclude list is [], retry=0 2024-12-04T19:26:28,062 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:28,062 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:28,062 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:28,064 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43389463/hregion-43389463.1733340388048 2024-12-04T19:26:28,064 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:28,064 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2013b2410e4ac6c577de1598238d3cb2, NAME => 'testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741893_1071 (size=67) 2024-12-04T19:26:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741893_1071 (size=67) 2024-12-04T19:26:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741893_1071 (size=67) 2024-12-04T19:26:28,074 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:28,075 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,076 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName a 2024-12-04T19:26:28,076 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,077 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,077 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,078 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName b 2024-12-04T19:26:28,079 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,079 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,079 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,080 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName c 2024-12-04T19:26:28,080 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,081 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,081 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,083 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,083 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,084 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:28,085 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,087 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:28,087 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2013b2410e4ac6c577de1598238d3cb2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66386574, jitterRate=-0.010762959718704224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2013b2410e4ac6c577de1598238d3cb2: Writing region info on filesystem at 1733340388074Initializing all the Stores at 1733340388075 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388075Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388075Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388075Cleaning up temporary data from old regions at 1733340388083 (+8 ms)Region opened successfully at 1733340388088 (+5 ms) 2024-12-04T19:26:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2013b2410e4ac6c577de1598238d3cb2, disabling compactions & flushes 2024-12-04T19:26:28,088 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. after waiting 0 ms 2024-12-04T19:26:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,088 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2013b2410e4ac6c577de1598238d3cb2: Waiting for close lock at 1733340388088Disabling compacts and flushes for region at 1733340388088Disabling writes for close at 1733340388088Writing region close event to WAL at 1733340388088Closed at 1733340388088 2024-12-04T19:26:28,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741892_1070 (size=93) 2024-12-04T19:26:28,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741892_1070 (size=93) 2024-12-04T19:26:28,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741892_1070 (size=93) 2024-12-04T19:26:28,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:28,093 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43389463:(num 1733340388048) 2024-12-04T19:26:28,093 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:28,095 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:28,108 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095, exclude list is [], retry=0 2024-12-04T19:26:28,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:28,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:28,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:28,112 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 2024-12-04T19:26:28,113 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:28,113 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2013b2410e4ac6c577de1598238d3cb2, NAME => 'testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:28,113 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:28,113 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,113 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,114 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,115 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName a 2024-12-04T19:26:28,115 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,115 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,115 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,116 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName b 2024-12-04T19:26:28,116 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,117 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,117 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,117 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName c 2024-12-04T19:26:28,117 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,118 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,118 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,118 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,119 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,120 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,120 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,121 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,123 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2013b2410e4ac6c577de1598238d3cb2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62573624, jitterRate=-0.06758034229278564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:28,123 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2013b2410e4ac6c577de1598238d3cb2: Writing region info on filesystem at 1733340388113Initializing all the Stores at 1733340388114 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388114Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388114Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388114Cleaning up temporary data from old regions at 1733340388120 (+6 ms)Region opened successfully at 1733340388123 (+3 ms) 2024-12-04T19:26:28,149 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2013b2410e4ac6c577de1598238d3cb2 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T19:26:28,164 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/a/b52a9f35e512471ea3e1e432a19f9223 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733340388123/Put/seqid=0 2024-12-04T19:26:28,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741895_1073 (size=5958) 2024-12-04T19:26:28,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741895_1073 (size=5958) 2024-12-04T19:26:28,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741895_1073 (size=5958) 2024-12-04T19:26:28,171 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/a/b52a9f35e512471ea3e1e432a19f9223 2024-12-04T19:26:28,189 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/b/21b5fbfeb5cb44bcaf592d5c5cb510d7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733340388131/Put/seqid=0 2024-12-04T19:26:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741896_1074 (size=5958) 2024-12-04T19:26:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741896_1074 (size=5958) 2024-12-04T19:26:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741896_1074 (size=5958) 2024-12-04T19:26:28,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/b/21b5fbfeb5cb44bcaf592d5c5cb510d7 2024-12-04T19:26:28,218 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/c/91b736d238fc42149f86c785b9645770 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733340388139/Put/seqid=0 2024-12-04T19:26:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741897_1075 (size=5958) 2024-12-04T19:26:28,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741897_1075 (size=5958) 2024-12-04T19:26:28,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741897_1075 (size=5958) 2024-12-04T19:26:28,224 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/c/91b736d238fc42149f86c785b9645770 2024-12-04T19:26:28,230 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/a/b52a9f35e512471ea3e1e432a19f9223 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/a/b52a9f35e512471ea3e1e432a19f9223 2024-12-04T19:26:28,236 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/a/b52a9f35e512471ea3e1e432a19f9223, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T19:26:28,237 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/b/21b5fbfeb5cb44bcaf592d5c5cb510d7 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/b/21b5fbfeb5cb44bcaf592d5c5cb510d7 2024-12-04T19:26:28,242 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/b/21b5fbfeb5cb44bcaf592d5c5cb510d7, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T19:26:28,243 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/c/91b736d238fc42149f86c785b9645770 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/c/91b736d238fc42149f86c785b9645770 2024-12-04T19:26:28,249 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/c/91b736d238fc42149f86c785b9645770, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T19:26:28,250 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2013b2410e4ac6c577de1598238d3cb2 in 101ms, sequenceid=33, compaction requested=false 2024-12-04T19:26:28,250 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2013b2410e4ac6c577de1598238d3cb2: 2024-12-04T19:26:28,250 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2013b2410e4ac6c577de1598238d3cb2, disabling compactions & flushes 2024-12-04T19:26:28,250 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,250 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,251 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. after waiting 0 ms 2024-12-04T19:26:28,251 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,252 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. 2024-12-04T19:26:28,252 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2013b2410e4ac6c577de1598238d3cb2: Waiting for close lock at 1733340388250Disabling compacts and flushes for region at 1733340388250Disabling writes for close at 1733340388251 (+1 ms)Writing region close event to WAL at 1733340388252 (+1 ms)Closed at 1733340388252 2024-12-04T19:26:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741894_1072 (size=2873) 2024-12-04T19:26:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741894_1072 (size=2873) 2024-12-04T19:26:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741894_1072 (size=2873) 2024-12-04T19:26:28,259 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/b/21b5fbfeb5cb44bcaf592d5c5cb510d7 to hdfs://localhost:33061/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/b/21b5fbfeb5cb44bcaf592d5c5cb510d7 2024-12-04T19:26:28,273 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095, size=2.8 K (2873bytes) 2024-12-04T19:26:28,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 2024-12-04T19:26:28,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 after 0ms 2024-12-04T19:26:28,275 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:28,275 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 took 2ms 2024-12-04T19:26:28,278 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 so closing down 2024-12-04T19:26:28,278 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:28,278 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340388095.temp 2024-12-04T19:26:28,279 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000003-wal.1733340388095.temp 2024-12-04T19:26:28,280 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:28,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741898_1076 (size=2312) 2024-12-04T19:26:28,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741898_1076 (size=2312) 2024-12-04T19:26:28,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741898_1076 (size=2312) 2024-12-04T19:26:28,286 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000003-wal.1733340388095.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:28,287 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000003-wal.1733340388095.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032 2024-12-04T19:26:28,287 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-04T19:26:28,287 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095, journal: Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095, size=2.8 K (2873bytes) at 1733340388273Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 so closing down at 1733340388278 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000003-wal.1733340388095.temp at 1733340388279 (+1 ms)3 split writer threads finished at 1733340388280 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000003-wal.1733340388095.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733340388286 (+6 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000003-wal.1733340388095.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032 at 1733340388287 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733340388287 2024-12-04T19:26:28,289 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388095 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340388095 2024-12-04T19:26:28,290 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032 2024-12-04T19:26:28,290 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:28,292 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:28,309 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388292, exclude list is [], retry=0 2024-12-04T19:26:28,312 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:28,312 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:28,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:28,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388292 2024-12-04T19:26:28,315 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:28,315 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2013b2410e4ac6c577de1598238d3cb2, NAME => 'testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:28,315 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:28,315 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,315 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,316 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,317 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName a 2024-12-04T19:26:28,317 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,322 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/a/b52a9f35e512471ea3e1e432a19f9223 2024-12-04T19:26:28,322 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,322 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,323 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName b 2024-12-04T19:26:28,324 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,324 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,324 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,325 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2013b2410e4ac6c577de1598238d3cb2 columnFamilyName c 2024-12-04T19:26:28,325 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,329 DEBUG [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/c/91b736d238fc42149f86c785b9645770 2024-12-04T19:26:28,330 INFO [StoreOpener-2013b2410e4ac6c577de1598238d3cb2-1 {}] regionserver.HStore(327): Store=2013b2410e4ac6c577de1598238d3cb2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,330 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,333 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032 2024-12-04T19:26:28,335 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:28,336 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032 2024-12-04T19:26:28,336 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2013b2410e4ac6c577de1598238d3cb2 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T19:26:28,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/b/8016ba67af184de088369febe4659233 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733340388131/Put/seqid=0 2024-12-04T19:26:28,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741900_1078 (size=5958) 2024-12-04T19:26:28,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741900_1078 (size=5958) 2024-12-04T19:26:28,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741900_1078 (size=5958) 2024-12-04T19:26:28,359 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/b/8016ba67af184de088369febe4659233 2024-12-04T19:26:28,366 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/.tmp/b/8016ba67af184de088369febe4659233 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/b/8016ba67af184de088369febe4659233 2024-12-04T19:26:28,371 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/b/8016ba67af184de088369febe4659233, entries=10, sequenceid=32, filesize=5.8 K 2024-12-04T19:26:28,371 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2013b2410e4ac6c577de1598238d3cb2 in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-12-04T19:26:28,372 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/0000000000000000032 2024-12-04T19:26:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,374 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:28,375 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2013b2410e4ac6c577de1598238d3cb2 2024-12-04T19:26:28,377 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/2013b2410e4ac6c577de1598238d3cb2/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-04T19:26:28,378 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2013b2410e4ac6c577de1598238d3cb2; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59242732, jitterRate=-0.11721450090408325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:28,378 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2013b2410e4ac6c577de1598238d3cb2: Writing region info on filesystem at 1733340388315Initializing all the Stores at 1733340388316 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388316Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388316Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388316Obtaining lock to block concurrent updates at 1733340388336 (+20 ms)Preparing flush snapshotting stores in 2013b2410e4ac6c577de1598238d3cb2 at 1733340388336Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733340388336Flushing stores of testReplayEditsWrittenViaHRegion,,1733340388043.2013b2410e4ac6c577de1598238d3cb2. at 1733340388336Flushing 2013b2410e4ac6c577de1598238d3cb2/b: creating writer at 1733340388336Flushing 2013b2410e4ac6c577de1598238d3cb2/b: appending metadata at 1733340388351 (+15 ms)Flushing 2013b2410e4ac6c577de1598238d3cb2/b: closing flushed file at 1733340388351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e8d286b: reopening flushed file at 1733340388365 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2013b2410e4ac6c577de1598238d3cb2 in 35ms, sequenceid=32, compaction requested=false; wal=null at 1733340388371 (+6 ms)Cleaning up temporary data from old regions at 1733340388373 (+2 ms)Region opened successfully at 1733340388378 (+5 ms) 2024-12-04T19:26:28,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=421 (was 412) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:32776 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:60842 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:53614 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:36644 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:36496 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:53734 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 277), ProcessCount=11 (was 11), AvailableMemoryMB=15669 (was 15675) 2024-12-04T19:26:28,400 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-04T19:26:28,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=421, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=15668 2024-12-04T19:26:28,412 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-04T19:26:28,426 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:28,428 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:28,429 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:28,431 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13005751, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-13005751, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:28,442 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13005751/hregion-13005751.1733340388431, exclude list is [], retry=0 2024-12-04T19:26:28,444 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:28,445 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:28,445 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:28,447 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13005751/hregion-13005751.1733340388431 2024-12-04T19:26:28,447 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:28,447 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 661667ab2ea4b68ba7fce49935cca818, NAME => 'testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:28,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741902_1080 (size=68) 2024-12-04T19:26:28,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741902_1080 (size=68) 2024-12-04T19:26:28,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741902_1080 (size=68) 2024-12-04T19:26:28,456 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:28,457 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,458 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName a 2024-12-04T19:26:28,458 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,459 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,459 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,460 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName b 2024-12-04T19:26:28,460 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,460 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,460 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,462 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName c 2024-12-04T19:26:28,462 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,462 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,462 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,464 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,465 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,465 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,465 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:28,467 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,469 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:28,469 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 661667ab2ea4b68ba7fce49935cca818; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64357038, jitterRate=-0.04100540280342102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:28,470 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 661667ab2ea4b68ba7fce49935cca818: Writing region info on filesystem at 1733340388456Initializing all the Stores at 1733340388457 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388457Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388457Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388457Cleaning up temporary data from old regions at 1733340388465 (+8 ms)Region opened successfully at 1733340388470 (+5 ms) 2024-12-04T19:26:28,470 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 661667ab2ea4b68ba7fce49935cca818, disabling compactions & flushes 2024-12-04T19:26:28,470 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:28,471 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:28,471 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. after waiting 0 ms 2024-12-04T19:26:28,471 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:28,471 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:28,471 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 661667ab2ea4b68ba7fce49935cca818: Waiting for close lock at 1733340388470Disabling compacts and flushes for region at 1733340388470Disabling writes for close at 1733340388471 (+1 ms)Writing region close event to WAL at 1733340388471Closed at 1733340388471 2024-12-04T19:26:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741901_1079 (size=93) 2024-12-04T19:26:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741901_1079 (size=93) 2024-12-04T19:26:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741901_1079 (size=93) 2024-12-04T19:26:28,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:28,475 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13005751:(num 1733340388431) 2024-12-04T19:26:28,475 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:28,477 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:28,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478, exclude list is [], retry=0 2024-12-04T19:26:28,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:28,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:28,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:28,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 2024-12-04T19:26:28,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:28,569 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 661667ab2ea4b68ba7fce49935cca818, NAME => 'testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:28,572 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,572 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:28,572 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,572 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,577 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,578 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName a 2024-12-04T19:26:28,579 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,579 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,579 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,580 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName b 2024-12-04T19:26:28,581 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,581 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,581 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,582 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName c 2024-12-04T19:26:28,582 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:28,583 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:28,583 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,583 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,585 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,586 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,586 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,586 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:28,588 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,589 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 661667ab2ea4b68ba7fce49935cca818; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65059207, jitterRate=-0.030542269349098206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:28,589 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:28,590 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 661667ab2ea4b68ba7fce49935cca818: Running coprocessor pre-open hook at 1733340388572Writing region info on filesystem at 1733340388572Initializing all the Stores at 1733340388573 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388573Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388577 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340388577Cleaning up temporary data from old regions at 1733340388586 (+9 ms)Running coprocessor post-open hooks at 1733340388589 (+3 ms)Region opened successfully at 1733340388590 (+1 ms) 2024-12-04T19:26:28,600 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 661667ab2ea4b68ba7fce49935cca818 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-04T19:26:28,602 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:28,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-04T19:26:28,918 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:28,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-04T19:26:28,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:28,925 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T19:26:28,925 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:28,927 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-04T19:26:28,927 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:28,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-04T19:26:28,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:29,602 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:30,584 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:26:30,603 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:31,604 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:32,605 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:33,606 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:34,607 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:35,607 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:36,609 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:37,609 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 661667ab2ea4b68ba7fce49935cca818/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:37,611 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 661667ab2ea4b68ba7fce49935cca818: 2024-12-04T19:26:37,611 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:37,631 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 661667ab2ea4b68ba7fce49935cca818: 2024-12-04T19:26:37,631 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-04T19:26:37,631 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 661667ab2ea4b68ba7fce49935cca818, disabling compactions & flushes 2024-12-04T19:26:37,631 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:37,631 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:37,631 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. after waiting 0 ms 2024-12-04T19:26:37,631 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:37,632 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:37,632 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. 2024-12-04T19:26:37,632 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 661667ab2ea4b68ba7fce49935cca818: Waiting for close lock at 1733340397631Running coprocessor pre-close hooks at 1733340397631Disabling compacts and flushes for region at 1733340397631Disabling writes for close at 1733340397631Writing region close event to WAL at 1733340397632 (+1 ms)Running coprocessor post-close hooks at 1733340397632Closed at 1733340397632 2024-12-04T19:26:37,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741903_1081 (size=2398) 2024-12-04T19:26:37,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741903_1081 (size=2398) 2024-12-04T19:26:37,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741903_1081 (size=2398) 2024-12-04T19:26:37,651 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478, size=2.3 K (2398bytes) 2024-12-04T19:26:37,651 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 2024-12-04T19:26:37,651 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 after 0ms 2024-12-04T19:26:37,653 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:37,654 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 took 4ms 2024-12-04T19:26:37,655 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 so closing down 2024-12-04T19:26:37,655 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:37,657 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733340388478.temp 2024-12-04T19:26:37,658 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000004-wal.1733340388478.temp 2024-12-04T19:26:37,658 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:37,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741904_1082 (size=1672) 2024-12-04T19:26:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741904_1082 (size=1672) 2024-12-04T19:26:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741904_1082 (size=1672) 2024-12-04T19:26:37,664 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000004-wal.1733340388478.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:37,666 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000004-wal.1733340388478.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026 2024-12-04T19:26:37,666 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-04T19:26:37,666 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478, journal: Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478, size=2.3 K (2398bytes) at 1733340397651Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 so closing down at 1733340397655 (+4 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000004-wal.1733340388478.temp at 1733340397658 (+3 ms)3 split writer threads finished at 1733340397658Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000004-wal.1733340388478.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733340397664 (+6 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000004-wal.1733340388478.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026 at 1733340397666 (+2 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733340397666 2024-12-04T19:26:37,668 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340388478 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340388478 2024-12-04T19:26:37,668 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026 2024-12-04T19:26:37,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:37,670 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:37,681 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340397670, exclude list is [], retry=0 2024-12-04T19:26:37,683 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:37,683 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:37,684 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:37,685 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340397670 2024-12-04T19:26:37,685 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:37,686 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 661667ab2ea4b68ba7fce49935cca818, NAME => 'testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:37,686 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,686 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:37,686 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,686 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,688 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,689 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName a 2024-12-04T19:26:37,689 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:37,689 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:37,689 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,690 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName b 2024-12-04T19:26:37,690 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:37,691 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:37,691 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,692 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 661667ab2ea4b68ba7fce49935cca818 columnFamilyName c 2024-12-04T19:26:37,692 DEBUG [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:37,692 INFO [StoreOpener-661667ab2ea4b68ba7fce49935cca818-1 {}] regionserver.HStore(327): Store=661667ab2ea4b68ba7fce49935cca818/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:37,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,695 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026 2024-12-04T19:26:37,697 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:37,698 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026 2024-12-04T19:26:37,698 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 661667ab2ea4b68ba7fce49935cca818 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-04T19:26:37,711 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/a/b1ad1f377a8642caabb656a737292810 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733340397621/Put/seqid=0 2024-12-04T19:26:37,712 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:26:37,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741906_1084 (size=5523) 2024-12-04T19:26:37,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741906_1084 (size=5523) 2024-12-04T19:26:37,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741906_1084 (size=5523) 2024-12-04T19:26:37,717 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/a/b1ad1f377a8642caabb656a737292810 2024-12-04T19:26:37,735 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/b/67fd6f5255a6473cb4bcb54f6a3a006f is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733340397612/Put/seqid=0 2024-12-04T19:26:37,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741907_1085 (size=5524) 2024-12-04T19:26:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741907_1085 (size=5524) 2024-12-04T19:26:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741907_1085 (size=5524) 2024-12-04T19:26:37,741 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/b/67fd6f5255a6473cb4bcb54f6a3a006f 2024-12-04T19:26:37,761 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/c/7bf8c94cb6834ba2b871727787ffbe56 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733340397616/Put/seqid=0 2024-12-04T19:26:37,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741908_1086 (size=5457) 2024-12-04T19:26:37,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741908_1086 (size=5457) 2024-12-04T19:26:37,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741908_1086 (size=5457) 2024-12-04T19:26:37,768 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/c/7bf8c94cb6834ba2b871727787ffbe56 2024-12-04T19:26:37,775 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/a/b1ad1f377a8642caabb656a737292810 as hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/a/b1ad1f377a8642caabb656a737292810 2024-12-04T19:26:37,781 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/a/b1ad1f377a8642caabb656a737292810, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T19:26:37,782 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/b/67fd6f5255a6473cb4bcb54f6a3a006f as hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/b/67fd6f5255a6473cb4bcb54f6a3a006f 2024-12-04T19:26:37,788 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/b/67fd6f5255a6473cb4bcb54f6a3a006f, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T19:26:37,789 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/.tmp/c/7bf8c94cb6834ba2b871727787ffbe56 as hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/c/7bf8c94cb6834ba2b871727787ffbe56 2024-12-04T19:26:37,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/c/7bf8c94cb6834ba2b871727787ffbe56, entries=6, sequenceid=26, filesize=5.3 K 2024-12-04T19:26:37,795 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 661667ab2ea4b68ba7fce49935cca818 in 97ms, sequenceid=26, compaction requested=false; wal=null 2024-12-04T19:26:37,796 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/0000000000000000026 2024-12-04T19:26:37,797 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,797 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,798 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:37,799 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,801 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsAfterAbortingFlush/661667ab2ea4b68ba7fce49935cca818/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-04T19:26:37,803 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 661667ab2ea4b68ba7fce49935cca818; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65075190, jitterRate=-0.03030410408973694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:37,803 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 661667ab2ea4b68ba7fce49935cca818 2024-12-04T19:26:37,803 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 661667ab2ea4b68ba7fce49935cca818: Running coprocessor pre-open hook at 1733340397686Writing region info on filesystem at 1733340397686Initializing all the Stores at 1733340397687 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340397687Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340397687Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340397687Obtaining lock to block concurrent updates at 1733340397698 (+11 ms)Preparing flush snapshotting stores in 661667ab2ea4b68ba7fce49935cca818 at 1733340397698Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733340397699 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733340388426.661667ab2ea4b68ba7fce49935cca818. at 1733340397699Flushing 661667ab2ea4b68ba7fce49935cca818/a: creating writer at 1733340397699Flushing 661667ab2ea4b68ba7fce49935cca818/a: appending metadata at 1733340397711 (+12 ms)Flushing 661667ab2ea4b68ba7fce49935cca818/a: closing flushed file at 1733340397711Flushing 661667ab2ea4b68ba7fce49935cca818/b: creating writer at 1733340397722 (+11 ms)Flushing 661667ab2ea4b68ba7fce49935cca818/b: appending metadata at 1733340397734 (+12 ms)Flushing 661667ab2ea4b68ba7fce49935cca818/b: closing flushed file at 1733340397734Flushing 661667ab2ea4b68ba7fce49935cca818/c: creating writer at 1733340397747 (+13 ms)Flushing 661667ab2ea4b68ba7fce49935cca818/c: appending metadata at 1733340397760 (+13 ms)Flushing 661667ab2ea4b68ba7fce49935cca818/c: closing flushed file at 1733340397760Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ceb6cdc: reopening flushed file at 1733340397774 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@221548e6: reopening flushed file at 1733340397781 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@134a8a55: reopening flushed file at 1733340397788 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 661667ab2ea4b68ba7fce49935cca818 in 97ms, sequenceid=26, compaction requested=false; wal=null at 1733340397795 (+7 ms)Cleaning up temporary data from old regions at 1733340397797 (+2 ms)Running coprocessor post-open hooks at 1733340397803 (+6 ms)Region opened successfully at 1733340397803 2024-12-04T19:26:37,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=417 (was 421), OpenFileDescriptor=1181 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 277), ProcessCount=11 (was 11), AvailableMemoryMB=15620 (was 15668) 2024-12-04T19:26:37,825 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1181 is superior to 1024 2024-12-04T19:26:37,836 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1181, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=15618 2024-12-04T19:26:37,836 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1181 is superior to 1024 2024-12-04T19:26:37,852 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:37,854 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:37,854 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:37,857 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13704701, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-13704701, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:37,869 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13704701/hregion-13704701.1733340397857, exclude list is [], retry=0 2024-12-04T19:26:37,872 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:37,872 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:37,873 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:37,874 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13704701/hregion-13704701.1733340397857 2024-12-04T19:26:37,875 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:37,875 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 574563a8d3f7d04d60b38f53b9e802b1, NAME => 'testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741910_1088 (size=61) 2024-12-04T19:26:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741910_1088 (size=61) 2024-12-04T19:26:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741910_1088 (size=61) 2024-12-04T19:26:37,885 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:37,886 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,887 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 574563a8d3f7d04d60b38f53b9e802b1 columnFamilyName a 2024-12-04T19:26:37,887 DEBUG [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:37,888 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(327): Store=574563a8d3f7d04d60b38f53b9e802b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:37,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,888 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,889 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,889 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,889 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,890 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,892 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:37,893 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 574563a8d3f7d04d60b38f53b9e802b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61234197, jitterRate=-0.08753935992717743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:37,893 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 574563a8d3f7d04d60b38f53b9e802b1: Writing region info on filesystem at 1733340397885Initializing all the Stores at 1733340397885Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340397885Cleaning up temporary data from old regions at 1733340397889 (+4 ms)Region opened successfully at 1733340397893 (+4 ms) 2024-12-04T19:26:37,894 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 574563a8d3f7d04d60b38f53b9e802b1, disabling compactions & flushes 2024-12-04T19:26:37,894 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,894 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,894 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. after waiting 0 ms 2024-12-04T19:26:37,894 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,896 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,896 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 574563a8d3f7d04d60b38f53b9e802b1: Waiting for close lock at 1733340397894Disabling compacts and flushes for region at 1733340397894Disabling writes for close at 1733340397894Writing region close event to WAL at 1733340397896 (+2 ms)Closed at 1733340397896 2024-12-04T19:26:37,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741909_1087 (size=93) 2024-12-04T19:26:37,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741909_1087 (size=93) 2024-12-04T19:26:37,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741909_1087 (size=93) 2024-12-04T19:26:37,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:37,902 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13704701:(num 1733340397857) 2024-12-04T19:26:37,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:37,904 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:37,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904, exclude list is [], retry=0 2024-12-04T19:26:37,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:37,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:37,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:37,920 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 2024-12-04T19:26:37,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:37,921 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 574563a8d3f7d04d60b38f53b9e802b1, NAME => 'testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:37,921 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:37,921 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,921 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,922 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,923 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 574563a8d3f7d04d60b38f53b9e802b1 columnFamilyName a 2024-12-04T19:26:37,923 DEBUG [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:37,923 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(327): Store=574563a8d3f7d04d60b38f53b9e802b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:37,923 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,924 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,925 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,925 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,925 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,927 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:37,928 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 574563a8d3f7d04d60b38f53b9e802b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69812011, jitterRate=0.04028002917766571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:37,928 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 574563a8d3f7d04d60b38f53b9e802b1: Writing region info on filesystem at 1733340397921Initializing all the Stores at 1733340397922 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340397922Cleaning up temporary data from old regions at 1733340397925 (+3 ms)Region opened successfully at 1733340397928 (+3 ms) 2024-12-04T19:26:37,936 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 574563a8d3f7d04d60b38f53b9e802b1, disabling compactions & flushes 2024-12-04T19:26:37,936 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,936 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,936 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. after waiting 0 ms 2024-12-04T19:26:37,936 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,937 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,937 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. 2024-12-04T19:26:37,937 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 574563a8d3f7d04d60b38f53b9e802b1: Waiting for close lock at 1733340397936Disabling compacts and flushes for region at 1733340397936Disabling writes for close at 1733340397936Writing region close event to WAL at 1733340397937 (+1 ms)Closed at 1733340397937 2024-12-04T19:26:37,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741911_1089 (size=838) 2024-12-04T19:26:37,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741911_1089 (size=838) 2024-12-04T19:26:37,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741911_1089 (size=838) 2024-12-04T19:26:37,953 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904, size=838 (838bytes) 2024-12-04T19:26:37,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 2024-12-04T19:26:37,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 after 0ms 2024-12-04T19:26:37,956 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:37,956 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 took 3ms 2024-12-04T19:26:37,958 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 so closing down 2024-12-04T19:26:37,958 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:37,959 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340397904.temp 2024-12-04T19:26:37,960 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000003-wal.1733340397904.temp 2024-12-04T19:26:37,960 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:37,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741912_1090 (size=838) 2024-12-04T19:26:37,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741912_1090 (size=838) 2024-12-04T19:26:37,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741912_1090 (size=838) 2024-12-04T19:26:37,969 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000003-wal.1733340397904.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:37,970 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000003-wal.1733340397904.temp to hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012 2024-12-04T19:26:37,970 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904, size=838, length=838, corrupted=false, cancelled=false 2024-12-04T19:26:37,970 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904, journal: Splitting hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904, size=838 (838bytes) at 1733340397953Finishing writing output for hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 so closing down at 1733340397958 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000003-wal.1733340397904.temp at 1733340397960 (+2 ms)3 split writer threads finished at 1733340397961 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000003-wal.1733340397904.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733340397969 (+8 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000003-wal.1733340397904.temp to hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012 at 1733340397970 (+1 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904, size=838, length=838, corrupted=false, cancelled=false at 1733340397970 2024-12-04T19:26:37,972 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340397904 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340397904 2024-12-04T19:26:37,973 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012 2024-12-04T19:26:37,976 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:38,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:38,264 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:38,274 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340398264, exclude list is [], retry=0 2024-12-04T19:26:38,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:38,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:38,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:38,278 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340398264 2024-12-04T19:26:38,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:38,278 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 574563a8d3f7d04d60b38f53b9e802b1, NAME => 'testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:38,278 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:38,278 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,278 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,281 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,282 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 574563a8d3f7d04d60b38f53b9e802b1 columnFamilyName a 2024-12-04T19:26:38,282 DEBUG [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,283 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(327): Store=574563a8d3f7d04d60b38f53b9e802b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,283 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,286 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,286 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012 2024-12-04T19:26:38,288 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:38,289 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012 2024-12-04T19:26:38,290 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 574563a8d3f7d04d60b38f53b9e802b1 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-04T19:26:38,312 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/.tmp/a/83c40a2d7e554220b92dded1f9bb3199 is 79, key is testDatalossWhenInputError/a:x0/1733340397928/Put/seqid=0 2024-12-04T19:26:38,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741914_1092 (size=5808) 2024-12-04T19:26:38,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741914_1092 (size=5808) 2024-12-04T19:26:38,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741914_1092 (size=5808) 2024-12-04T19:26:38,318 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/.tmp/a/83c40a2d7e554220b92dded1f9bb3199 2024-12-04T19:26:38,327 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/.tmp/a/83c40a2d7e554220b92dded1f9bb3199 as hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/a/83c40a2d7e554220b92dded1f9bb3199 2024-12-04T19:26:38,337 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/a/83c40a2d7e554220b92dded1f9bb3199, entries=10, sequenceid=12, filesize=5.7 K 2024-12-04T19:26:38,337 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 574563a8d3f7d04d60b38f53b9e802b1 in 48ms, sequenceid=12, compaction requested=false; wal=null 2024-12-04T19:26:38,338 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/0000000000000000012 2024-12-04T19:26:38,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,342 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,345 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-04T19:26:38,346 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 574563a8d3f7d04d60b38f53b9e802b1; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69876565, jitterRate=0.04124195873737335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:38,347 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 574563a8d3f7d04d60b38f53b9e802b1: Writing region info on filesystem at 1733340398278Initializing all the Stores at 1733340398280 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398280Obtaining lock to block concurrent updates at 1733340398290 (+10 ms)Preparing flush snapshotting stores in 574563a8d3f7d04d60b38f53b9e802b1 at 1733340398290Finished memstore snapshotting testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733340398290Flushing stores of testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1. at 1733340398290Flushing 574563a8d3f7d04d60b38f53b9e802b1/a: creating writer at 1733340398290Flushing 574563a8d3f7d04d60b38f53b9e802b1/a: appending metadata at 1733340398311 (+21 ms)Flushing 574563a8d3f7d04d60b38f53b9e802b1/a: closing flushed file at 1733340398311Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57c7b219: reopening flushed file at 1733340398326 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 574563a8d3f7d04d60b38f53b9e802b1 in 48ms, sequenceid=12, compaction requested=false; wal=null at 1733340398337 (+11 ms)Cleaning up temporary data from old regions at 1733340398339 (+2 ms)Region opened successfully at 1733340398347 (+8 ms) 2024-12-04T19:26:38,350 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 574563a8d3f7d04d60b38f53b9e802b1, NAME => 'testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:38,351 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340397852.574563a8d3f7d04d60b38f53b9e802b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:38,351 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,351 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,352 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,354 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 574563a8d3f7d04d60b38f53b9e802b1 columnFamilyName a 2024-12-04T19:26:38,354 DEBUG [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,360 DEBUG [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/a/83c40a2d7e554220b92dded1f9bb3199 2024-12-04T19:26:38,361 INFO [StoreOpener-574563a8d3f7d04d60b38f53b9e802b1-1 {}] regionserver.HStore(327): Store=574563a8d3f7d04d60b38f53b9e802b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,361 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,362 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,363 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,364 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,364 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,366 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 574563a8d3f7d04d60b38f53b9e802b1 2024-12-04T19:26:38,368 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testDatalossWhenInputError/574563a8d3f7d04d60b38f53b9e802b1/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-04T19:26:38,369 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 574563a8d3f7d04d60b38f53b9e802b1; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69858943, jitterRate=0.04097937047481537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:26:38,370 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 574563a8d3f7d04d60b38f53b9e802b1: Writing region info on filesystem at 1733340398351Initializing all the Stores at 1733340398352 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398352Cleaning up temporary data from old regions at 1733340398364 (+12 ms)Region opened successfully at 1733340398370 (+6 ms) 2024-12-04T19:26:38,386 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=427 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:39412 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54696 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:37404 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54622 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:39452 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-994869438-172.17.0.2-1733340363677:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1263 (was 1181) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=15612 (was 15618) 2024-12-04T19:26:38,386 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1263 is superior to 1024 2024-12-04T19:26:38,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=427, OpenFileDescriptor=1263, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=15611 2024-12-04T19:26:38,399 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1263 is superior to 1024 2024-12-04T19:26:38,411 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:38,413 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:38,413 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:38,416 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64064326, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-64064326, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:38,427 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64064326/hregion-64064326.1733340398416, exclude list is [], retry=0 2024-12-04T19:26:38,430 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:38,430 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:38,431 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:38,432 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64064326/hregion-64064326.1733340398416 2024-12-04T19:26:38,433 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:38,433 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c02870f91ad36656133a5f14e522b201, NAME => 'testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:38,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741916_1094 (size=63) 2024-12-04T19:26:38,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741916_1094 (size=63) 2024-12-04T19:26:38,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741916_1094 (size=63) 2024-12-04T19:26:38,448 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:38,449 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,450 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName a 2024-12-04T19:26:38,450 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,451 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,451 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,452 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName b 2024-12-04T19:26:38,452 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,452 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,453 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,454 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName c 2024-12-04T19:26:38,454 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,455 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,455 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,456 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,456 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,457 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:38,458 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,460 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:38,460 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c02870f91ad36656133a5f14e522b201; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72800298, jitterRate=0.08480897545814514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:38,461 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c02870f91ad36656133a5f14e522b201: Writing region info on filesystem at 1733340398448Initializing all the Stores at 1733340398449 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398449Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398449Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398449Cleaning up temporary data from old regions at 1733340398457 (+8 ms)Region opened successfully at 1733340398461 (+4 ms) 2024-12-04T19:26:38,461 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c02870f91ad36656133a5f14e522b201, disabling compactions & flushes 2024-12-04T19:26:38,461 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:38,461 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:38,461 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. after waiting 0 ms 2024-12-04T19:26:38,461 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:38,461 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:38,461 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c02870f91ad36656133a5f14e522b201: Waiting for close lock at 1733340398461Disabling compacts and flushes for region at 1733340398461Disabling writes for close at 1733340398461Writing region close event to WAL at 1733340398461Closed at 1733340398461 2024-12-04T19:26:38,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741915_1093 (size=93) 2024-12-04T19:26:38,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741915_1093 (size=93) 2024-12-04T19:26:38,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741915_1093 (size=93) 2024-12-04T19:26:38,467 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:38,467 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64064326:(num 1733340398416) 2024-12-04T19:26:38,468 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:38,469 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:38,481 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469, exclude list is [], retry=0 2024-12-04T19:26:38,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:38,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:38,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:38,485 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 2024-12-04T19:26:38,485 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:38,485 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c02870f91ad36656133a5f14e522b201, NAME => 'testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:38,486 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:38,486 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,486 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,487 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,488 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName a 2024-12-04T19:26:38,488 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,489 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,489 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,490 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName b 2024-12-04T19:26:38,490 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,490 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,491 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,492 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName c 2024-12-04T19:26:38,492 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:38,492 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:38,493 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,493 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,494 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,495 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,495 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,496 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:38,497 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:38,498 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c02870f91ad36656133a5f14e522b201; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69020273, jitterRate=0.028482213616371155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:38,498 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c02870f91ad36656133a5f14e522b201: Writing region info on filesystem at 1733340398486Initializing all the Stores at 1733340398487 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398487Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398487Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340398487Cleaning up temporary data from old regions at 1733340398495 (+8 ms)Region opened successfully at 1733340398498 (+3 ms) 2024-12-04T19:26:38,505 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733340398504/Put/seqid=0 2024-12-04T19:26:38,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741918_1096 (size=4875) 2024-12-04T19:26:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741918_1096 (size=4875) 2024-12-04T19:26:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741918_1096 (size=4875) 2024-12-04T19:26:38,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733340398516/Put/seqid=0 2024-12-04T19:26:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741919_1097 (size=4875) 2024-12-04T19:26:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741919_1097 (size=4875) 2024-12-04T19:26:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741919_1097 (size=4875) 2024-12-04T19:26:38,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733340398526/Put/seqid=0 2024-12-04T19:26:38,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741920_1098 (size=4875) 2024-12-04T19:26:38,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741920_1098 (size=4875) 2024-12-04T19:26:38,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741920_1098 (size=4875) 2024-12-04T19:26:38,536 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c02870f91ad36656133a5f14e522b201/a 2024-12-04T19:26:38,540 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-04T19:26:38,540 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:26:38,540 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c02870f91ad36656133a5f14e522b201/a 2024-12-04T19:26:38,544 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-04T19:26:38,544 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:26:38,544 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c02870f91ad36656133a5f14e522b201/a 2024-12-04T19:26:38,548 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-04T19:26:38,548 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:26:38,548 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c02870f91ad36656133a5f14e522b201 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-04T19:26:38,565 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/.tmp/a/1891d0dba13b4b54ba6118d9de197094 is 55, key is testCompactedBulkLoadedFiles/a:a/1733340398499/Put/seqid=0 2024-12-04T19:26:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741921_1099 (size=5107) 2024-12-04T19:26:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741921_1099 (size=5107) 2024-12-04T19:26:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741921_1099 (size=5107) 2024-12-04T19:26:38,572 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/.tmp/a/1891d0dba13b4b54ba6118d9de197094 2024-12-04T19:26:38,578 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/.tmp/a/1891d0dba13b4b54ba6118d9de197094 as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094 2024-12-04T19:26:38,584 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094, entries=1, sequenceid=4, filesize=5.0 K 2024-12-04T19:26:38,585 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c02870f91ad36656133a5f14e522b201 in 37ms, sequenceid=4, compaction requested=false 2024-12-04T19:26:38,585 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c02870f91ad36656133a5f14e522b201: 2024-12-04T19:26:38,586 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ 2024-12-04T19:26:38,587 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ 2024-12-04T19:26:38,588 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ 2024-12-04T19:26:38,589 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile0 into c02870f91ad36656133a5f14e522b201/a as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ - updating store file list. 2024-12-04T19:26:38,593 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fe13fa207e484614a2a3ed3386fcd988_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:38,593 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ into c02870f91ad36656133a5f14e522b201/a 2024-12-04T19:26:38,593 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile0 into c02870f91ad36656133a5f14e522b201/a (new location: hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_) 2024-12-04T19:26:38,594 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile1 into c02870f91ad36656133a5f14e522b201/a as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ - updating store file list. 2024-12-04T19:26:38,597 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9ab2376da2f2412aa812a3983192e56e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:38,598 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ into c02870f91ad36656133a5f14e522b201/a 2024-12-04T19:26:38,598 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile1 into c02870f91ad36656133a5f14e522b201/a (new location: hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_) 2024-12-04T19:26:38,598 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile2 into c02870f91ad36656133a5f14e522b201/a as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ - updating store file list. 2024-12-04T19:26:38,602 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for adf46899526349d093d89ba4fc913e22_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:38,602 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ into c02870f91ad36656133a5f14e522b201/a 2024-12-04T19:26:38,602 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33061/hbase/testCompactedBulkLoadedFiles/hfile2 into c02870f91ad36656133a5f14e522b201/a (new location: hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_) 2024-12-04T19:26:38,609 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:26:38,609 DEBUG [Time-limited test {}] regionserver.HStore(1541): c02870f91ad36656133a5f14e522b201/a is initiating major compaction (all files) 2024-12-04T19:26:38,609 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c02870f91ad36656133a5f14e522b201/a in testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:38,610 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_] into tmpdir=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/.tmp, totalSize=19.3 K 2024-12-04T19:26:38,610 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1891d0dba13b4b54ba6118d9de197094, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733340398499 2024-12-04T19:26:38,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fe13fa207e484614a2a3ed3386fcd988_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T19:26:38,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9ab2376da2f2412aa812a3983192e56e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T19:26:38,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting adf46899526349d093d89ba4fc913e22_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T19:26:38,624 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/.tmp/a/83ca877412c54845974aecd011039269 is 55, key is testCompactedBulkLoadedFiles/a:a/1733340398499/Put/seqid=0 2024-12-04T19:26:38,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741922_1100 (size=6154) 2024-12-04T19:26:38,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741922_1100 (size=6154) 2024-12-04T19:26:38,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741922_1100 (size=6154) 2024-12-04T19:26:38,636 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/.tmp/a/83ca877412c54845974aecd011039269 as hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/83ca877412c54845974aecd011039269 2024-12-04T19:26:38,642 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c02870f91ad36656133a5f14e522b201/a of c02870f91ad36656133a5f14e522b201 into 83ca877412c54845974aecd011039269(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:26:38,643 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c02870f91ad36656133a5f14e522b201: 2024-12-04T19:26:38,643 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T19:26:38,643 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T19:26:38,670 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469, size=0 (0bytes) 2024-12-04T19:26:38,670 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 might be still open, length is 0 2024-12-04T19:26:38,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 2024-12-04T19:26:38,671 WARN [IPC Server handler 4 on default port 33061 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-04T19:26:38,671 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 after 1ms 2024-12-04T19:26:38,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-04T19:26:38,918 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:38,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-04T19:26:38,920 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-04T19:26:40,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:39486 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39486 dst: /127.0.0.1:45577 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45577 remote=/127.0.0.1:39486]. Total timeout mills is 60000, 57995 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:40,650 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:37436 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41073:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37436 dst: /127.0.0.1:41073 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:40,650 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54734 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:42863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54734 dst: /127.0.0.1:42863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:40,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741917_1101 (size=1368) 2024-12-04T19:26:40,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741917_1101 (size=1368) 2024-12-04T19:26:42,672 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 after 4002ms 2024-12-04T19:26:42,675 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:42,675 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 took 4005ms 2024-12-04T19:26:42,676 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469; continuing. 2024-12-04T19:26:42,677 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 so closing down 2024-12-04T19:26:42,677 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:42,678 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340398469.temp 2024-12-04T19:26:42,679 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000003-wal.1733340398469.temp 2024-12-04T19:26:42,679 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:42,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741923_1102 (size=635) 2024-12-04T19:26:42,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741923_1102 (size=635) 2024-12-04T19:26:42,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741923_1102 (size=635) 2024-12-04T19:26:42,689 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000003-wal.1733340398469.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:42,691 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000003-wal.1733340398469.temp to hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008 2024-12-04T19:26:42,691 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:26:42,691 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469, journal: Splitting hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469, size=0 (0bytes) at 1733340398670Finishing writing output for hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 so closing down at 1733340402677 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000003-wal.1733340398469.temp at 1733340402679 (+2 ms)3 split writer threads finished at 1733340402679Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000003-wal.1733340398469.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733340402689 (+10 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000003-wal.1733340398469.temp to hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008 at 1733340402691 (+2 ms)Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469, size=0, length=0, corrupted=false, cancelled=false at 1733340402691 2024-12-04T19:26:42,693 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340398469 2024-12-04T19:26:42,694 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008 2024-12-04T19:26:42,694 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:42,695 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:42,706 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340402696, exclude list is [], retry=0 2024-12-04T19:26:42,708 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:42,709 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:42,709 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:42,710 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340402696 2024-12-04T19:26:42,710 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:42,710 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c02870f91ad36656133a5f14e522b201, NAME => 'testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:42,711 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:42,711 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,711 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,712 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,713 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName a 2024-12-04T19:26:42,713 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,718 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094 2024-12-04T19:26:42,721 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/83ca877412c54845974aecd011039269 2024-12-04T19:26:42,725 DEBUG [StoreFileOpener-c02870f91ad36656133a5f14e522b201-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9ab2376da2f2412aa812a3983192e56e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:42,725 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ 2024-12-04T19:26:42,728 DEBUG [StoreFileOpener-c02870f91ad36656133a5f14e522b201-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for adf46899526349d093d89ba4fc913e22_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:42,728 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ 2024-12-04T19:26:42,732 DEBUG [StoreFileOpener-c02870f91ad36656133a5f14e522b201-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fe13fa207e484614a2a3ed3386fcd988_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:26:42,732 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ 2024-12-04T19:26:42,732 WARN [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@288859a3 2024-12-04T19:26:42,732 WARN [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@288859a3 2024-12-04T19:26:42,732 WARN [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@288859a3 2024-12-04T19:26:42,732 WARN [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@288859a3 2024-12-04T19:26:42,732 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_] to archive 2024-12-04T19:26:42,733 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:26:42,735 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094 to hdfs://localhost:33061/hbase/archive/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/1891d0dba13b4b54ba6118d9de197094 2024-12-04T19:26:42,736 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ to hdfs://localhost:33061/hbase/archive/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/9ab2376da2f2412aa812a3983192e56e_SeqId_4_ 2024-12-04T19:26:42,738 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ to hdfs://localhost:33061/hbase/archive/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/adf46899526349d093d89ba4fc913e22_SeqId_4_ 2024-12-04T19:26:42,739 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ to hdfs://localhost:33061/hbase/archive/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/a/fe13fa207e484614a2a3ed3386fcd988_SeqId_4_ 2024-12-04T19:26:42,739 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,739 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,740 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName b 2024-12-04T19:26:42,740 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,740 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,741 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,741 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02870f91ad36656133a5f14e522b201 columnFamilyName c 2024-12-04T19:26:42,741 DEBUG [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,742 INFO [StoreOpener-c02870f91ad36656133a5f14e522b201-1 {}] regionserver.HStore(327): Store=c02870f91ad36656133a5f14e522b201/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,742 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,743 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,745 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008 2024-12-04T19:26:42,746 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:42,748 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c02870f91ad36656133a5f14e522b201 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c02870f91ad36656133a5f14e522b201" family_name: "a" compaction_input: "1891d0dba13b4b54ba6118d9de197094" compaction_input: "fe13fa207e484614a2a3ed3386fcd988_SeqId_4_" compaction_input: "9ab2376da2f2412aa812a3983192e56e_SeqId_4_" compaction_input: "adf46899526349d093d89ba4fc913e22_SeqId_4_" compaction_output: "83ca877412c54845974aecd011039269" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-04T19:26:42,748 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-04T19:26:42,748 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008 2024-12-04T19:26:42,749 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/0000000000000000008 2024-12-04T19:26:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,751 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:42,752 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c02870f91ad36656133a5f14e522b201 2024-12-04T19:26:42,754 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testCompactedBulkLoadedFiles/c02870f91ad36656133a5f14e522b201/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T19:26:42,755 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c02870f91ad36656133a5f14e522b201; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75411732, jitterRate=0.1237223744392395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:42,756 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c02870f91ad36656133a5f14e522b201: Writing region info on filesystem at 1733340402711Initializing all the Stores at 1733340402712 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402712Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402712Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402712Cleaning up temporary data from old regions at 1733340402750 (+38 ms)Region opened successfully at 1733340402756 (+6 ms) 2024-12-04T19:26:42,758 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c02870f91ad36656133a5f14e522b201, disabling compactions & flushes 2024-12-04T19:26:42,758 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:42,758 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:42,758 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. after waiting 0 ms 2024-12-04T19:26:42,758 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:42,759 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733340398411.c02870f91ad36656133a5f14e522b201. 2024-12-04T19:26:42,759 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c02870f91ad36656133a5f14e522b201: Waiting for close lock at 1733340402758Disabling compacts and flushes for region at 1733340402758Disabling writes for close at 1733340402758Writing region close event to WAL at 1733340402759 (+1 ms)Closed at 1733340402759 2024-12-04T19:26:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741924_1103 (size=93) 2024-12-04T19:26:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741924_1103 (size=93) 2024-12-04T19:26:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741924_1103 (size=93) 2024-12-04T19:26:42,764 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:42,765 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340402696) 2024-12-04T19:26:42,776 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=441 (was 427) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33061 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1011256234_22 at /127.0.0.1:45834 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1011256234_22 at /127.0.0.1:57182 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33061 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1011256234_22 at /127.0.0.1:54160 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1263) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=15595 (was 15611) 2024-12-04T19:26:42,776 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-04T19:26:42,788 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=441, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=15594 2024-12-04T19:26:42,788 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-04T19:26:42,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:42,802 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:26:42,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:26:42,804 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92425252, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/hregion-92425252, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:42,815 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92425252/hregion-92425252.1733340402805, exclude list is [], retry=0 2024-12-04T19:26:42,817 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:42,817 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:42,817 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:42,819 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92425252/hregion-92425252.1733340402805 2024-12-04T19:26:42,819 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:46817:46817)] 2024-12-04T19:26:42,819 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 650ccc42aec71da366aae6d2373915f2, NAME => 'testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33061/hbase 2024-12-04T19:26:42,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741926_1105 (size=67) 2024-12-04T19:26:42,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741926_1105 (size=67) 2024-12-04T19:26:42,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741926_1105 (size=67) 2024-12-04T19:26:42,829 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:42,830 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,831 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName a 2024-12-04T19:26:42,831 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,831 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,831 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,832 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName b 2024-12-04T19:26:42,833 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,833 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,833 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,834 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName c 2024-12-04T19:26:42,834 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,835 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,835 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,835 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,836 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,837 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,837 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,837 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:42,838 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,840 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:26:42,841 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 650ccc42aec71da366aae6d2373915f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64976377, jitterRate=-0.03177653253078461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:42,841 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 650ccc42aec71da366aae6d2373915f2: Writing region info on filesystem at 1733340402829Initializing all the Stores at 1733340402829Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402830 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402830Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402830Cleaning up temporary data from old regions at 1733340402837 (+7 ms)Region opened successfully at 1733340402841 (+4 ms) 2024-12-04T19:26:42,841 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 650ccc42aec71da366aae6d2373915f2, disabling compactions & flushes 2024-12-04T19:26:42,841 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,841 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,841 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. after waiting 0 ms 2024-12-04T19:26:42,841 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,841 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,841 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 650ccc42aec71da366aae6d2373915f2: Waiting for close lock at 1733340402841Disabling compacts and flushes for region at 1733340402841Disabling writes for close at 1733340402841Writing region close event to WAL at 1733340402841Closed at 1733340402841 2024-12-04T19:26:42,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741925_1104 (size=93) 2024-12-04T19:26:42,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741925_1104 (size=93) 2024-12-04T19:26:42,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741925_1104 (size=93) 2024-12-04T19:26:42,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:42,845 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92425252:(num 1733340402805) 2024-12-04T19:26:42,845 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:42,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:42,859 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847, exclude list is [], retry=0 2024-12-04T19:26:42,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:42,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:42,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:42,863 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 2024-12-04T19:26:42,864 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:42,864 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 650ccc42aec71da366aae6d2373915f2, NAME => 'testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:42,864 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:42,864 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,864 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,866 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,866 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName a 2024-12-04T19:26:42,866 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,867 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,867 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,868 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName b 2024-12-04T19:26:42,868 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,868 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,868 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,869 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName c 2024-12-04T19:26:42,869 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,869 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,869 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,871 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,872 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,872 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,872 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:42,874 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,874 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 650ccc42aec71da366aae6d2373915f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68664589, jitterRate=0.023182108998298645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:42,875 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 650ccc42aec71da366aae6d2373915f2: Writing region info on filesystem at 1733340402864Initializing all the Stores at 1733340402865 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402865Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402866 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402866Cleaning up temporary data from old regions at 1733340402872 (+6 ms)Region opened successfully at 1733340402875 (+3 ms) 2024-12-04T19:26:42,881 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 650ccc42aec71da366aae6d2373915f2 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T19:26:42,894 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/a/623fac07bc6e4dcca27f1aa311eff9b1 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733340402875/Put/seqid=0 2024-12-04T19:26:42,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741928_1107 (size=5958) 2024-12-04T19:26:42,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741928_1107 (size=5958) 2024-12-04T19:26:42,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741928_1107 (size=5958) 2024-12-04T19:26:42,900 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/a/623fac07bc6e4dcca27f1aa311eff9b1 2024-12-04T19:26:42,905 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/a/623fac07bc6e4dcca27f1aa311eff9b1 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/a/623fac07bc6e4dcca27f1aa311eff9b1 2024-12-04T19:26:42,910 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/a/623fac07bc6e4dcca27f1aa311eff9b1, entries=10, sequenceid=13, filesize=5.8 K 2024-12-04T19:26:42,911 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 650ccc42aec71da366aae6d2373915f2 in 30ms, sequenceid=13, compaction requested=false 2024-12-04T19:26:42,911 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 650ccc42aec71da366aae6d2373915f2: 2024-12-04T19:26:42,927 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 650ccc42aec71da366aae6d2373915f2, disabling compactions & flushes 2024-12-04T19:26:42,927 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,927 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,927 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. after waiting 0 ms 2024-12-04T19:26:42,927 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,928 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,928 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:42,928 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 650ccc42aec71da366aae6d2373915f2: Waiting for close lock at 1733340402927Disabling compacts and flushes for region at 1733340402927Disabling writes for close at 1733340402927Writing region close event to WAL at 1733340402928 (+1 ms)Closed at 1733340402928 2024-12-04T19:26:42,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741927_1106 (size=2805) 2024-12-04T19:26:42,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741927_1106 (size=2805) 2024-12-04T19:26:42,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741927_1106 (size=2805) 2024-12-04T19:26:42,945 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847, size=2.7 K (2805bytes) 2024-12-04T19:26:42,945 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 2024-12-04T19:26:42,945 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 after 0ms 2024-12-04T19:26:42,947 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:42,947 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 took 3ms 2024-12-04T19:26:42,948 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 so closing down 2024-12-04T19:26:42,948 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:42,949 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340402847.temp 2024-12-04T19:26:42,950 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000003-wal.1733340402847.temp 2024-12-04T19:26:42,950 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:42,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741929_1108 (size=2312) 2024-12-04T19:26:42,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741929_1108 (size=2312) 2024-12-04T19:26:42,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741929_1108 (size=2312) 2024-12-04T19:26:42,957 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000003-wal.1733340402847.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:42,959 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000003-wal.1733340402847.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035 2024-12-04T19:26:42,959 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-04T19:26:42,959 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847, journal: Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847, size=2.7 K (2805bytes) at 1733340402945Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 so closing down at 1733340402948 (+3 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000003-wal.1733340402847.temp at 1733340402950 (+2 ms)3 split writer threads finished at 1733340402950Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000003-wal.1733340402847.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733340402957 (+7 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000003-wal.1733340402847.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035 at 1733340402959 (+2 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733340402959 2024-12-04T19:26:42,961 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402847 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340402847 2024-12-04T19:26:42,961 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035 2024-12-04T19:26:42,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:42,963 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:42,978 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964, exclude list is [], retry=0 2024-12-04T19:26:42,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:42,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:42,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:42,982 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 2024-12-04T19:26:42,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833),(127.0.0.1/127.0.0.1:34841:34841)] 2024-12-04T19:26:42,983 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 650ccc42aec71da366aae6d2373915f2, NAME => 'testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:26:42,983 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:42,983 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,983 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,985 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,986 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName a 2024-12-04T19:26:42,986 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,991 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/a/623fac07bc6e4dcca27f1aa311eff9b1 2024-12-04T19:26:42,992 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,992 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,993 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName b 2024-12-04T19:26:42,993 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,993 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,993 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,994 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName c 2024-12-04T19:26:42,994 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:42,994 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:42,994 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,995 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,996 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:42,997 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035 2024-12-04T19:26:42,998 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:42,999 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035 2024-12-04T19:26:43,000 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 650ccc42aec71da366aae6d2373915f2 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-04T19:26:43,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/b/2c96b2c821444864ac9b76a633517785 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733340402911/Put/seqid=0 2024-12-04T19:26:43,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741931_1110 (size=5958) 2024-12-04T19:26:43,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741931_1110 (size=5958) 2024-12-04T19:26:43,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741931_1110 (size=5958) 2024-12-04T19:26:43,021 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/b/2c96b2c821444864ac9b76a633517785 2024-12-04T19:26:43,038 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/c/7e227953c2604edfb7c0cda601739381 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733340402917/Put/seqid=0 2024-12-04T19:26:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741932_1111 (size=5958) 2024-12-04T19:26:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741932_1111 (size=5958) 2024-12-04T19:26:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741932_1111 (size=5958) 2024-12-04T19:26:43,045 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/c/7e227953c2604edfb7c0cda601739381 2024-12-04T19:26:43,050 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/b/2c96b2c821444864ac9b76a633517785 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/b/2c96b2c821444864ac9b76a633517785 2024-12-04T19:26:43,057 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/b/2c96b2c821444864ac9b76a633517785, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T19:26:43,058 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/c/7e227953c2604edfb7c0cda601739381 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/c/7e227953c2604edfb7c0cda601739381 2024-12-04T19:26:43,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/c/7e227953c2604edfb7c0cda601739381, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T19:26:43,064 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 650ccc42aec71da366aae6d2373915f2 in 64ms, sequenceid=35, compaction requested=false; wal=null 2024-12-04T19:26:43,065 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000035 2024-12-04T19:26:43,066 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:43,066 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:43,066 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:43,067 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:43,069 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-04T19:26:43,070 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 650ccc42aec71da366aae6d2373915f2; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71074551, jitterRate=0.05909334123134613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:43,070 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 650ccc42aec71da366aae6d2373915f2: Writing region info on filesystem at 1733340402983Initializing all the Stores at 1733340402984 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402984Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402985 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340402985Obtaining lock to block concurrent updates at 1733340403000 (+15 ms)Preparing flush snapshotting stores in 650ccc42aec71da366aae6d2373915f2 at 1733340403000Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733340403000Flushing stores of testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. at 1733340403000Flushing 650ccc42aec71da366aae6d2373915f2/b: creating writer at 1733340403000Flushing 650ccc42aec71da366aae6d2373915f2/b: appending metadata at 1733340403014 (+14 ms)Flushing 650ccc42aec71da366aae6d2373915f2/b: closing flushed file at 1733340403015 (+1 ms)Flushing 650ccc42aec71da366aae6d2373915f2/c: creating writer at 1733340403025 (+10 ms)Flushing 650ccc42aec71da366aae6d2373915f2/c: appending metadata at 1733340403037 (+12 ms)Flushing 650ccc42aec71da366aae6d2373915f2/c: closing flushed file at 1733340403037Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f9038fd: reopening flushed file at 1733340403050 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@405cf197: reopening flushed file at 1733340403057 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 650ccc42aec71da366aae6d2373915f2 in 64ms, sequenceid=35, compaction requested=false; wal=null at 1733340403064 (+7 ms)Cleaning up temporary data from old regions at 1733340403066 (+2 ms)Region opened successfully at 1733340403070 (+4 ms) 2024-12-04T19:26:43,126 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964, size=0 (0bytes) 2024-12-04T19:26:43,126 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 might be still open, length is 0 2024-12-04T19:26:43,126 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 2024-12-04T19:26:43,127 WARN [IPC Server handler 4 on default port 33061 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-04T19:26:43,127 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 after 1ms 2024-12-04T19:26:43,150 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:26:43,650 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:45888 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41073:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45888 dst: /127.0.0.1:41073 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41073 remote=/127.0.0.1:45888]. Total timeout mills is 60000, 59447 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:43,651 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:54238 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54238 dst: /127.0.0.1:45577 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:43,651 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_687652391_22 at /127.0.0.1:57228 [Receiving block BP-994869438-172.17.0.2-1733340363677:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:42863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57228 dst: /127.0.0.1:42863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:43,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741930_1112 (size=2304) 2024-12-04T19:26:43,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741930_1112 (size=2304) 2024-12-04T19:26:43,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741930_1112 (size=2304) 2024-12-04T19:26:47,128 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 after 4002ms 2024-12-04T19:26:47,133 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:47,133 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 took 4007ms 2024-12-04T19:26:47,137 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964; continuing. 2024-12-04T19:26:47,137 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 so closing down 2024-12-04T19:26:47,137 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:26:47,140 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733340402964.temp 2024-12-04T19:26:47,141 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000037-wal.1733340402964.temp 2024-12-04T19:26:47,141 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:26:47,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741933_1113 (size=2312) 2024-12-04T19:26:47,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741933_1113 (size=2312) 2024-12-04T19:26:47,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741933_1113 (size=2312) 2024-12-04T19:26:47,148 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000037-wal.1733340402964.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T19:26:47,149 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000037-wal.1733340402964.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066 2024-12-04T19:26:47,149 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:26:47,150 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964, journal: Splitting hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964, size=0 (0bytes) at 1733340403126Finishing writing output for hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 so closing down at 1733340407137 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000037-wal.1733340402964.temp at 1733340407141 (+4 ms)3 split writer threads finished at 1733340407141Closed recovered edits writer path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000037-wal.1733340402964.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733340407148 (+7 ms)Rename recovered edits hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000037-wal.1733340402964.temp to hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066 at 1733340407149 (+1 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964, size=0, length=0, corrupted=false, cancelled=false at 1733340407149 2024-12-04T19:26:47,151 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 to hdfs://localhost:33061/hbase/oldWALs/wal.1733340402964 2024-12-04T19:26:47,152 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066 2024-12-04T19:26:47,152 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:26:47,154 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33061/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799, archiveDir=hdfs://localhost:33061/hbase/oldWALs, maxLogs=32 2024-12-04T19:26:47,167 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340407154, exclude list is [], retry=0 2024-12-04T19:26:47,169 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42863,DS-c8037e4c-2b4e-44d7-8ccd-5110d8f7d53c,DISK] 2024-12-04T19:26:47,169 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41073,DS-7ad38888-fe30-40c5-bb9a-922421de4bec,DISK] 2024-12-04T19:26:47,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-3f695e16-c624-48eb-9ee2-40e814f34b7a,DISK] 2024-12-04T19:26:47,171 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340407154 2024-12-04T19:26:47,171 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34841:34841),(127.0.0.1/127.0.0.1:46817:46817),(127.0.0.1/127.0.0.1:39833:39833)] 2024-12-04T19:26:47,172 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:26:47,173 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,174 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName a 2024-12-04T19:26:47,174 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:47,178 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/a/623fac07bc6e4dcca27f1aa311eff9b1 2024-12-04T19:26:47,178 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:47,178 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,179 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName b 2024-12-04T19:26:47,179 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:47,183 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/b/2c96b2c821444864ac9b76a633517785 2024-12-04T19:26:47,183 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:47,183 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,184 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 650ccc42aec71da366aae6d2373915f2 columnFamilyName c 2024-12-04T19:26:47,184 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:26:47,190 DEBUG [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/c/7e227953c2604edfb7c0cda601739381 2024-12-04T19:26:47,190 INFO [StoreOpener-650ccc42aec71da366aae6d2373915f2-1 {}] regionserver.HStore(327): Store=650ccc42aec71da366aae6d2373915f2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:26:47,190 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,193 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,193 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066 2024-12-04T19:26:47,195 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T19:26:47,199 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066 2024-12-04T19:26:47,199 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 650ccc42aec71da366aae6d2373915f2 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T19:26:47,217 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/a/0134440430f04a009d90bd8d4b70c935 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733340403077/Put/seqid=0 2024-12-04T19:26:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741935_1115 (size=5958) 2024-12-04T19:26:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741935_1115 (size=5958) 2024-12-04T19:26:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741935_1115 (size=5958) 2024-12-04T19:26:47,223 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/a/0134440430f04a009d90bd8d4b70c935 2024-12-04T19:26:47,239 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/b/cdc80048ddcb4ebeafb447792cbc6bbe is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733340403085/Put/seqid=0 2024-12-04T19:26:47,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741936_1116 (size=5958) 2024-12-04T19:26:47,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741936_1116 (size=5958) 2024-12-04T19:26:47,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741936_1116 (size=5958) 2024-12-04T19:26:47,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/b/cdc80048ddcb4ebeafb447792cbc6bbe 2024-12-04T19:26:47,261 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/c/243eb2ded7d0417399d80ddd5bcfaf11 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733340403091/Put/seqid=0 2024-12-04T19:26:47,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741937_1117 (size=5958) 2024-12-04T19:26:47,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741937_1117 (size=5958) 2024-12-04T19:26:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741937_1117 (size=5958) 2024-12-04T19:26:47,267 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/c/243eb2ded7d0417399d80ddd5bcfaf11 2024-12-04T19:26:47,272 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/a/0134440430f04a009d90bd8d4b70c935 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/a/0134440430f04a009d90bd8d4b70c935 2024-12-04T19:26:47,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/a/0134440430f04a009d90bd8d4b70c935, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T19:26:47,278 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/b/cdc80048ddcb4ebeafb447792cbc6bbe as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/b/cdc80048ddcb4ebeafb447792cbc6bbe 2024-12-04T19:26:47,282 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/b/cdc80048ddcb4ebeafb447792cbc6bbe, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T19:26:47,283 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/.tmp/c/243eb2ded7d0417399d80ddd5bcfaf11 as hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/c/243eb2ded7d0417399d80ddd5bcfaf11 2024-12-04T19:26:47,287 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/c/243eb2ded7d0417399d80ddd5bcfaf11, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T19:26:47,287 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 650ccc42aec71da366aae6d2373915f2 in 88ms, sequenceid=66, compaction requested=false; wal=null 2024-12-04T19:26:47,288 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/0000000000000000066 2024-12-04T19:26:47,289 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,289 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,289 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:26:47,290 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 650ccc42aec71da366aae6d2373915f2 2024-12-04T19:26:47,292 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/hbase/data/default/testReplayEditsWrittenViaHRegion/650ccc42aec71da366aae6d2373915f2/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-04T19:26:47,293 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 650ccc42aec71da366aae6d2373915f2; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66695756, jitterRate=-0.006155788898468018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:26:47,293 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 650ccc42aec71da366aae6d2373915f2: Writing region info on filesystem at 1733340407172Initializing all the Stores at 1733340407173 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340407173Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340407173Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340407173Obtaining lock to block concurrent updates at 1733340407199 (+26 ms)Preparing flush snapshotting stores in 650ccc42aec71da366aae6d2373915f2 at 1733340407199Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733340407199Flushing stores of testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. at 1733340407199Flushing 650ccc42aec71da366aae6d2373915f2/a: creating writer at 1733340407199Flushing 650ccc42aec71da366aae6d2373915f2/a: appending metadata at 1733340407216 (+17 ms)Flushing 650ccc42aec71da366aae6d2373915f2/a: closing flushed file at 1733340407216Flushing 650ccc42aec71da366aae6d2373915f2/b: creating writer at 1733340407226 (+10 ms)Flushing 650ccc42aec71da366aae6d2373915f2/b: appending metadata at 1733340407238 (+12 ms)Flushing 650ccc42aec71da366aae6d2373915f2/b: closing flushed file at 1733340407238Flushing 650ccc42aec71da366aae6d2373915f2/c: creating writer at 1733340407249 (+11 ms)Flushing 650ccc42aec71da366aae6d2373915f2/c: appending metadata at 1733340407260 (+11 ms)Flushing 650ccc42aec71da366aae6d2373915f2/c: closing flushed file at 1733340407260Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@226cd450: reopening flushed file at 1733340407271 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16e55fe1: reopening flushed file at 1733340407277 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29488a63: reopening flushed file at 1733340407282 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 650ccc42aec71da366aae6d2373915f2 in 88ms, sequenceid=66, compaction requested=false; wal=null at 1733340407287 (+5 ms)Cleaning up temporary data from old regions at 1733340407289 (+2 ms)Region opened successfully at 1733340407293 (+4 ms) 2024-12-04T19:26:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 650ccc42aec71da366aae6d2373915f2, disabling compactions & flushes 2024-12-04T19:26:47,305 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. after waiting 0 ms 2024-12-04T19:26:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:47,337 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340402800.650ccc42aec71da366aae6d2373915f2. 2024-12-04T19:26:47,338 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 650ccc42aec71da366aae6d2373915f2: Waiting for close lock at 1733340407305Disabling compacts and flushes for region at 1733340407305Disabling writes for close at 1733340407305Writing region close event to WAL at 1733340407337 (+32 ms)Closed at 1733340407337 2024-12-04T19:26:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741934_1114 (size=93) 2024-12-04T19:26:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741934_1114 (size=93) 2024-12-04T19:26:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741934_1114 (size=93) 2024-12-04T19:26:47,344 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:26:47,344 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340407154) 2024-12-04T19:26:47,358 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=447 (was 441) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33061 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-955431554_22 at /127.0.0.1:45968 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-955431554_22 at /127.0.0.1:54260 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33061 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-955431554_22 at /127.0.0.1:57270 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=212 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=15570 (was 15594) 2024-12-04T19:26:47,358 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-12-04T19:26:47,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T19:26:47,358 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:26:47,358 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:26:47,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:26:47,359 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T19:26:47,359 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=954470236, stopped=false 2024-12-04T19:26:47,360 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9ba2734cd39f,42915,1733340368520 2024-12-04T19:26:47,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:47,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:47,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:26:47,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:47,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:47,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:47,413 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T19:26:47,414 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:26:47,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:26:47,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:47,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:47,415 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,41753,1733340369441' ***** 2024-12-04T19:26:47,415 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T19:26:47,415 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,34937,1733340369505' ***** 2024-12-04T19:26:47,415 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T19:26:47,415 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:26:47,415 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:26:47,415 INFO [RS:2;9ba2734cd39f:34937 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T19:26:47,415 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:26:47,415 INFO [RS:2;9ba2734cd39f:34937 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T19:26:47,415 INFO [RS:1;9ba2734cd39f:41753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T19:26:47,415 INFO [RS:1;9ba2734cd39f:41753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T19:26:47,415 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(959): stopping server 9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:47,416 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(3091): Received CLOSE for ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:47,416 INFO [RS:2;9ba2734cd39f:34937 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9ba2734cd39f:34937. 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(959): stopping server 9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:47,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:26:47,416 DEBUG [RS:2;9ba2734cd39f:34937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9ba2734cd39f:41753. 2024-12-04T19:26:47,416 DEBUG [RS:2;9ba2734cd39f:34937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,416 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef32ee4d500fb92aaaa8c18d0bf47143, disabling compactions & flushes 2024-12-04T19:26:47,416 DEBUG [RS:1;9ba2734cd39f:41753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:26:47,416 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,34937,1733340369505; all regions closed. 2024-12-04T19:26:47,416 DEBUG [RS:1;9ba2734cd39f:41753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,416 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:47,416 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:26:47,416 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. after waiting 0 ms 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:26:47,416 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:26:47,416 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:26:47,416 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T19:26:47,417 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T19:26:47,417 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1325): Online Regions={ef32ee4d500fb92aaaa8c18d0bf47143=testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T19:26:47,417 DEBUG [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ef32ee4d500fb92aaaa8c18d0bf47143 2024-12-04T19:26:47,417 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:26:47,417 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:26:47,417 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:26:47,417 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:26:47,417 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:26:47,417 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-04T19:26:47,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741835_1011 (size=93) 2024-12-04T19:26:47,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741835_1011 (size=93) 2024-12-04T19:26:47,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741835_1011 (size=93) 2024-12-04T19:26:47,423 DEBUG [RS:2;9ba2734cd39f:34937 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs 2024-12-04T19:26:47,423 INFO [RS:2;9ba2734cd39f:34937 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C34937%2C1733340369505:(num 1733340371226) 2024-12-04T19:26:47,423 DEBUG [RS:2;9ba2734cd39f:34937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,423 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:26:47,423 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:26:47,424 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T19:26:47,424 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:26:47,424 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:26:47,424 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:26:47,424 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:26:47,424 INFO [RS:2;9ba2734cd39f:34937 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34937 2024-12-04T19:26:47,424 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:26:47,425 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ef32ee4d500fb92aaaa8c18d0bf47143/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-04T19:26:47,425 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:47,426 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef32ee4d500fb92aaaa8c18d0bf47143: Waiting for close lock at 1733340407416Running coprocessor pre-close hooks at 1733340407416Disabling compacts and flushes for region at 1733340407416Disabling writes for close at 1733340407416Writing region close event to WAL at 1733340407420 (+4 ms)Running coprocessor post-close hooks at 1733340407425 (+5 ms)Closed at 1733340407425 2024-12-04T19:26:47,426 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143. 2024-12-04T19:26:47,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,34937,1733340369505 2024-12-04T19:26:47,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:26:47,434 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:26:47,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,34937,1733340369505] 2024-12-04T19:26:47,442 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/info/845a59a138c943bd8d4cda0f4505a342 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733340384647.ef32ee4d500fb92aaaa8c18d0bf47143./info:regioninfo/1733340387730/Put/seqid=0 2024-12-04T19:26:47,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741938_1118 (size=8243) 2024-12-04T19:26:47,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741938_1118 (size=8243) 2024-12-04T19:26:47,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741938_1118 (size=8243) 2024-12-04T19:26:47,452 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/info/845a59a138c943bd8d4cda0f4505a342 2024-12-04T19:26:47,455 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,34937,1733340369505 already deleted, retry=false 2024-12-04T19:26:47,455 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ba2734cd39f,34937,1733340369505 expired; onlineServers=1 2024-12-04T19:26:47,471 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/ns/cb21013ce4514374bd04887935b1b40e is 43, key is default/ns:d/1733340371940/Put/seqid=0 2024-12-04T19:26:47,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741939_1119 (size=5153) 2024-12-04T19:26:47,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741939_1119 (size=5153) 2024-12-04T19:26:47,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741939_1119 (size=5153) 2024-12-04T19:26:47,477 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/ns/cb21013ce4514374bd04887935b1b40e 2024-12-04T19:26:47,493 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/table/9dc2e43b3dc54c28b3a13c252f4b278e is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733340385067/Put/seqid=0 2024-12-04T19:26:47,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741940_1120 (size=5431) 2024-12-04T19:26:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741940_1120 (size=5431) 2024-12-04T19:26:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741940_1120 (size=5431) 2024-12-04T19:26:47,499 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/table/9dc2e43b3dc54c28b3a13c252f4b278e 2024-12-04T19:26:47,503 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:26:47,503 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:26:47,506 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/info/845a59a138c943bd8d4cda0f4505a342 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/info/845a59a138c943bd8d4cda0f4505a342 2024-12-04T19:26:47,512 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/info/845a59a138c943bd8d4cda0f4505a342, entries=18, sequenceid=21, filesize=8.0 K 2024-12-04T19:26:47,513 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/ns/cb21013ce4514374bd04887935b1b40e as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/ns/cb21013ce4514374bd04887935b1b40e 2024-12-04T19:26:47,519 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/ns/cb21013ce4514374bd04887935b1b40e, entries=2, sequenceid=21, filesize=5.0 K 2024-12-04T19:26:47,520 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/.tmp/table/9dc2e43b3dc54c28b3a13c252f4b278e as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/table/9dc2e43b3dc54c28b3a13c252f4b278e 2024-12-04T19:26:47,525 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/table/9dc2e43b3dc54c28b3a13c252f4b278e, entries=2, sequenceid=21, filesize=5.3 K 2024-12-04T19:26:47,526 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=21, compaction requested=false 2024-12-04T19:26:47,531 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-04T19:26:47,532 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:26:47,532 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:26:47,532 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733340407417Running coprocessor pre-close hooks at 1733340407417Disabling compacts and flushes for region at 1733340407417Disabling writes for close at 1733340407417Obtaining lock to block concurrent updates at 1733340407417Preparing flush snapshotting stores in 1588230740 at 1733340407417Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733340407421 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733340407422 (+1 ms)Flushing 1588230740/info: creating writer at 1733340407423 (+1 ms)Flushing 1588230740/info: appending metadata at 1733340407442 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733340407442Flushing 1588230740/ns: creating writer at 1733340407457 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733340407470 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733340407471 (+1 ms)Flushing 1588230740/table: creating writer at 1733340407481 (+10 ms)Flushing 1588230740/table: appending metadata at 1733340407492 (+11 ms)Flushing 1588230740/table: closing flushed file at 1733340407492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45c22940: reopening flushed file at 1733340407505 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11bf5195: reopening flushed file at 1733340407512 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cb6efa6: reopening flushed file at 1733340407519 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=21, compaction requested=false at 1733340407526 (+7 ms)Writing region close event to WAL at 1733340407528 (+2 ms)Running coprocessor post-close hooks at 1733340407532 (+4 ms)Closed at 1733340407532 2024-12-04T19:26:47,532 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T19:26:47,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:47,545 INFO [RS:2;9ba2734cd39f:34937 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:26:47,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34937-0x101a1cf074a0003, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:47,545 INFO [RS:2;9ba2734cd39f:34937 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,34937,1733340369505; zookeeper connection closed. 2024-12-04T19:26:47,545 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6127bb27 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6127bb27 2024-12-04T19:26:47,617 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,41753,1733340369441; all regions closed. 2024-12-04T19:26:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741836_1012 (size=3783) 2024-12-04T19:26:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741836_1012 (size=3783) 2024-12-04T19:26:47,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741836_1012 (size=3783) 2024-12-04T19:26:47,625 DEBUG [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs 2024-12-04T19:26:47,625 INFO [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C41753%2C1733340369441.meta:.meta(num 1733340371782) 2024-12-04T19:26:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741834_1010 (size=2652) 2024-12-04T19:26:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741834_1010 (size=2652) 2024-12-04T19:26:47,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741834_1010 (size=2652) 2024-12-04T19:26:47,631 DEBUG [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/oldWALs 2024-12-04T19:26:47,631 INFO [RS:1;9ba2734cd39f:41753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C41753%2C1733340369441:(num 1733340371226) 2024-12-04T19:26:47,631 DEBUG [RS:1;9ba2734cd39f:41753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:26:47,631 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:26:47,631 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:26:47,631 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T19:26:47,631 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:26:47,631 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:26:47,632 INFO [RS:1;9ba2734cd39f:41753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41753 2024-12-04T19:26:47,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:26:47,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,41753,1733340369441 2024-12-04T19:26:47,645 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:26:47,655 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,41753,1733340369441] 2024-12-04T19:26:47,666 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,41753,1733340369441 already deleted, retry=false 2024-12-04T19:26:47,666 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ba2734cd39f,41753,1733340369441 expired; onlineServers=0 2024-12-04T19:26:47,666 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9ba2734cd39f,42915,1733340368520' ***** 2024-12-04T19:26:47,666 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T19:26:47,666 INFO [M:0;9ba2734cd39f:42915 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:26:47,667 INFO [M:0;9ba2734cd39f:42915 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:26:47,667 DEBUG [M:0;9ba2734cd39f:42915 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T19:26:47,667 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T19:26:47,667 DEBUG [M:0;9ba2734cd39f:42915 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T19:26:47,667 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733340370674 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733340370674,5,FailOnTimeoutGroup] 2024-12-04T19:26:47,667 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733340370675 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733340370675,5,FailOnTimeoutGroup] 2024-12-04T19:26:47,668 INFO [M:0;9ba2734cd39f:42915 {}] hbase.ChoreService(370): Chore service for: master/9ba2734cd39f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T19:26:47,668 INFO [M:0;9ba2734cd39f:42915 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:26:47,668 DEBUG [M:0;9ba2734cd39f:42915 {}] master.HMaster(1795): Stopping service threads 2024-12-04T19:26:47,668 INFO [M:0;9ba2734cd39f:42915 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T19:26:47,669 INFO [M:0;9ba2734cd39f:42915 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T19:26:47,670 INFO [M:0;9ba2734cd39f:42915 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T19:26:47,670 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T19:26:47,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T19:26:47,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:26:47,677 DEBUG [M:0;9ba2734cd39f:42915 {}] zookeeper.ZKUtil(347): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T19:26:47,677 WARN [M:0;9ba2734cd39f:42915 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T19:26:47,678 INFO [M:0;9ba2734cd39f:42915 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/.lastflushedseqids 2024-12-04T19:26:47,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741941_1121 (size=138) 2024-12-04T19:26:47,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741941_1121 (size=138) 2024-12-04T19:26:47,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741941_1121 (size=138) 2024-12-04T19:26:47,693 INFO [M:0;9ba2734cd39f:42915 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T19:26:47,694 INFO [M:0;9ba2734cd39f:42915 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T19:26:47,694 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T19:26:47,694 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:47,694 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:47,694 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T19:26:47,694 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:47,694 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-12-04T19:26:47,707 DEBUG [M:0;9ba2734cd39f:42915 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d8d5696d6a3e42749aa8069f8db10321 is 82, key is hbase:meta,,1/info:regioninfo/1733340371864/Put/seqid=0 2024-12-04T19:26:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741942_1122 (size=5672) 2024-12-04T19:26:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741942_1122 (size=5672) 2024-12-04T19:26:47,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741942_1122 (size=5672) 2024-12-04T19:26:47,713 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d8d5696d6a3e42749aa8069f8db10321 2024-12-04T19:26:47,733 DEBUG [M:0;9ba2734cd39f:42915 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74d369a558724ad9be66298b92676915 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733340385073/Put/seqid=0 2024-12-04T19:26:47,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741943_1123 (size=7754) 2024-12-04T19:26:47,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741943_1123 (size=7754) 2024-12-04T19:26:47,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741943_1123 (size=7754) 2024-12-04T19:26:47,755 INFO [RS:1;9ba2734cd39f:41753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:26:47,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:47,755 INFO [RS:1;9ba2734cd39f:41753 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,41753,1733340369441; zookeeper connection closed. 2024-12-04T19:26:47,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41753-0x101a1cf074a0002, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:47,756 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b429fff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b429fff 2024-12-04T19:26:47,756 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T19:26:48,145 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74d369a558724ad9be66298b92676915 2024-12-04T19:26:48,155 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74d369a558724ad9be66298b92676915 2024-12-04T19:26:48,170 DEBUG [M:0;9ba2734cd39f:42915 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/99ab69dfbd6e4eb8b41be44ee954824c is 69, key is 9ba2734cd39f,34937,1733340369505/rs:state/1733340370850/Put/seqid=0 2024-12-04T19:26:48,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741944_1124 (size=5440) 2024-12-04T19:26:48,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741944_1124 (size=5440) 2024-12-04T19:26:48,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741944_1124 (size=5440) 2024-12-04T19:26:48,175 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/99ab69dfbd6e4eb8b41be44ee954824c 2024-12-04T19:26:48,179 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 99ab69dfbd6e4eb8b41be44ee954824c 2024-12-04T19:26:48,180 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d8d5696d6a3e42749aa8069f8db10321 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d8d5696d6a3e42749aa8069f8db10321 2024-12-04T19:26:48,185 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d8d5696d6a3e42749aa8069f8db10321, entries=8, sequenceid=168, filesize=5.5 K 2024-12-04T19:26:48,186 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74d369a558724ad9be66298b92676915 as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74d369a558724ad9be66298b92676915 2024-12-04T19:26:48,190 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74d369a558724ad9be66298b92676915 2024-12-04T19:26:48,190 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74d369a558724ad9be66298b92676915, entries=17, sequenceid=168, filesize=7.6 K 2024-12-04T19:26:48,191 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/99ab69dfbd6e4eb8b41be44ee954824c as hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/99ab69dfbd6e4eb8b41be44ee954824c 2024-12-04T19:26:48,194 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 99ab69dfbd6e4eb8b41be44ee954824c 2024-12-04T19:26:48,194 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33061/user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/99ab69dfbd6e4eb8b41be44ee954824c, entries=3, sequenceid=168, filesize=5.3 K 2024-12-04T19:26:48,195 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 501ms, sequenceid=168, compaction requested=false 2024-12-04T19:26:48,196 INFO [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:26:48,196 DEBUG [M:0;9ba2734cd39f:42915 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733340407694Disabling compacts and flushes for region at 1733340407694Disabling writes for close at 1733340407694Obtaining lock to block concurrent updates at 1733340407694Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733340407694Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1733340407694Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733340407695 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733340407695Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733340407706 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733340407706Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733340407716 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733340407732 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733340407732Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733340408155 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733340408169 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733340408169Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48f6c3b1: reopening flushed file at 1733340408179 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@828d52: reopening flushed file at 1733340408185 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8a30e3d: reopening flushed file at 1733340408190 (+5 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 501ms, sequenceid=168, compaction requested=false at 1733340408195 (+5 ms)Writing region close event to WAL at 1733340408196 (+1 ms)Closed at 1733340408196 2024-12-04T19:26:48,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741830_1006 (size=71325) 2024-12-04T19:26:48,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42863 is added to blk_1073741830_1006 (size=71325) 2024-12-04T19:26:48,198 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/8132825e-f7a6-ce2a-2d7e-ccedccb2fe90/MasterData/WALs/9ba2734cd39f,42915,1733340368520/9ba2734cd39f%2C42915%2C1733340368520.1733340369932 not finished, retry = 0 2024-12-04T19:26:48,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41073 is added to blk_1073741830_1006 (size=71325) 2024-12-04T19:26:48,300 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:26:48,300 INFO [M:0;9ba2734cd39f:42915 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T19:26:48,301 INFO [M:0;9ba2734cd39f:42915 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42915 2024-12-04T19:26:48,301 INFO [M:0;9ba2734cd39f:42915 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:26:48,451 INFO [M:0;9ba2734cd39f:42915 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:26:48,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:48,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42915-0x101a1cf074a0000, quorum=127.0.0.1:59927, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:26:48,462 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388292 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388292 (inode 16655) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340388042/wal.1733340388292 (inode 16655) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:26:48,463 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340380096/wal.1733340380329 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:48,463 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340402799/wal.1733340402964 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:48,466 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340397670 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340397670 (inode 16678) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340388425/wal.1733340397670 (inode 16678) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:26:48,467 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340373020/wal.1733340373083 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:48,469 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340398264 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340398264 (inode 16704) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340397851/wal.1733340398264 (inode 16704) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:26:48,469 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340398410/wal.1733340398469 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:26:48,471 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal.1733340372889 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal.1733340372889 (inode 16485) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340372698/wal.1733340372889 (inode 16485) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:26:48,474 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal.1733340372470 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal.1733340372470 (inode 16462) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340372267/wal.1733340372470 (inode 16462) Holder DFSClient_NONMAPREDUCE_687652391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:26:48,477 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:26:48,479 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:26:48,479 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:26:48,479 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:26:48,479 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,STOPPED} 2024-12-04T19:26:48,482 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:26:48,482 WARN [BP-994869438-172.17.0.2-1733340363677 heartbeating to localhost/127.0.0.1:33061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:26:48,482 WARN [BP-994869438-172.17.0.2-1733340363677 heartbeating to localhost/127.0.0.1:33061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-994869438-172.17.0.2-1733340363677 (Datanode Uuid 3ed7986c-c0b6-4e2e-b397-5b01cd5417cf) service to localhost/127.0.0.1:33061 2024-12-04T19:26:48,482 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:26:48,483 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data5/current/BP-994869438-172.17.0.2-1733340363677 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:26:48,483 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data6/current/BP-994869438-172.17.0.2-1733340363677 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:26:48,484 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:26:48,485 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:26:48,486 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:26:48,486 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:26:48,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:26:48,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,STOPPED} 2024-12-04T19:26:48,487 WARN [BP-994869438-172.17.0.2-1733340363677 heartbeating to localhost/127.0.0.1:33061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:26:48,487 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:26:48,487 WARN [BP-994869438-172.17.0.2-1733340363677 heartbeating to localhost/127.0.0.1:33061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-994869438-172.17.0.2-1733340363677 (Datanode Uuid 7869fb6a-3980-48db-b24c-8ee77fdbc783) service to localhost/127.0.0.1:33061 2024-12-04T19:26:48,487 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:26:48,487 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data3/current/BP-994869438-172.17.0.2-1733340363677 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:26:48,488 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data4/current/BP-994869438-172.17.0.2-1733340363677 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:26:48,488 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:26:48,495 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:26:48,495 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:26:48,495 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:26:48,495 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:26:48,495 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,STOPPED} 2024-12-04T19:26:48,496 WARN [BP-994869438-172.17.0.2-1733340363677 heartbeating to localhost/127.0.0.1:33061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:26:48,496 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:26:48,496 WARN [BP-994869438-172.17.0.2-1733340363677 heartbeating to localhost/127.0.0.1:33061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-994869438-172.17.0.2-1733340363677 (Datanode Uuid 11a6b42d-7789-4a67-9bb1-b5a2934cd644) service to localhost/127.0.0.1:33061 2024-12-04T19:26:48,496 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:26:48,497 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data1/current/BP-994869438-172.17.0.2-1733340363677 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:26:48,497 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/cluster_387e75c8-d3bd-8e31-d13b-7c3e08821e55/data/data2/current/BP-994869438-172.17.0.2-1733340363677 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:26:48,497 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:26:48,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T19:26:48,503 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:26:48,503 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:26:48,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:26:48,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/30187afe-b710-0567-c726-61897572d2e6/hadoop.log.dir/,STOPPED} 2024-12-04T19:26:48,511 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T19:26:48,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down