2024-11-24 13:27:24,024 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-24 13:27:24,038 main DEBUG Took 0.011883 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 13:27:24,039 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 13:27:24,039 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 13:27:24,040 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 13:27:24,042 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,050 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 13:27:24,065 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,067 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,068 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,069 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,070 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,071 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,073 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,074 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,075 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,076 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,076 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,077 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:27:24,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,080 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 13:27:24,081 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:27:24,083 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 13:27:24,084 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 13:27:24,085 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 13:27:24,087 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 13:27:24,087 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 13:27:24,097 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 13:27:24,100 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 13:27:24,101 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 13:27:24,101 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 13:27:24,102 main DEBUG createAppenders(={Console}) 2024-11-24 13:27:24,103 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-24 13:27:24,103 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-24 13:27:24,103 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-24 13:27:24,104 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 13:27:24,104 main DEBUG OutputStream closed 2024-11-24 13:27:24,105 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 13:27:24,105 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 13:27:24,106 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-24 13:27:24,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 13:27:24,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 13:27:24,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 13:27:24,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 13:27:24,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 13:27:24,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 13:27:24,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 13:27:24,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 13:27:24,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 13:27:24,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 13:27:24,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 13:27:24,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 13:27:24,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 13:27:24,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 13:27:24,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 13:27:24,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 13:27:24,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 13:27:24,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 13:27:24,200 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 13:27:24,200 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-24 13:27:24,201 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 13:27:24,201 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-24T13:27:24,445 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888 2024-11-24 13:27:24,448 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 13:27:24,448 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T13:27:24,459 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T13:27:24,484 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T13:27:24,532 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T13:27:24,533 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T13:27:24,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T13:27:24,561 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e, deleteOnExit=true 2024-11-24T13:27:24,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T13:27:24,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/test.cache.data in system properties and HBase conf 2024-11-24T13:27:24,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T13:27:24,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir in system properties and HBase conf 2024-11-24T13:27:24,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T13:27:24,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T13:27:24,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T13:27:24,656 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T13:27:24,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:27:24,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T13:27:24,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T13:27:24,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T13:27:24,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T13:27:24,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T13:27:24,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T13:27:24,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T13:27:24,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T13:27:24,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T13:27:24,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/nfs.dump.dir in system properties and HBase conf 2024-11-24T13:27:24,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/java.io.tmpdir in system properties and HBase conf 2024-11-24T13:27:24,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T13:27:24,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T13:27:24,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T13:27:25,571 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T13:27:25,664 INFO [Time-limited test {}] log.Log(170): Logging initialized @2377ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T13:27:25,750 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T13:27:25,809 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T13:27:25,830 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T13:27:25,830 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T13:27:25,831 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T13:27:25,844 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T13:27:25,847 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,AVAILABLE} 2024-11-24T13:27:25,848 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T13:27:26,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/java.io.tmpdir/jetty-localhost-38085-hadoop-hdfs-3_4_1-tests_jar-_-any-16985962617941986266/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T13:27:26,070 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:38085} 2024-11-24T13:27:26,070 INFO [Time-limited test {}] server.Server(415): Started @2784ms 2024-11-24T13:27:26,448 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T13:27:26,454 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T13:27:26,455 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T13:27:26,455 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T13:27:26,455 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T13:27:26,456 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,AVAILABLE} 2024-11-24T13:27:26,457 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T13:27:26,576 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/java.io.tmpdir/jetty-localhost-35079-hadoop-hdfs-3_4_1-tests_jar-_-any-11963450221261914808/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T13:27:26,577 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:35079} 2024-11-24T13:27:26,577 INFO [Time-limited test {}] server.Server(415): Started @3291ms 2024-11-24T13:27:26,632 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T13:27:26,748 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T13:27:26,753 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T13:27:26,757 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T13:27:26,757 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T13:27:26,757 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T13:27:26,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,AVAILABLE} 2024-11-24T13:27:26,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T13:27:26,883 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/java.io.tmpdir/jetty-localhost-43099-hadoop-hdfs-3_4_1-tests_jar-_-any-15883808098664989243/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T13:27:26,884 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:43099} 2024-11-24T13:27:26,884 INFO [Time-limited test {}] server.Server(415): Started @3598ms 2024-11-24T13:27:26,887 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T13:27:26,939 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T13:27:26,944 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T13:27:26,945 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T13:27:26,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T13:27:26,946 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T13:27:26,949 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,AVAILABLE} 2024-11-24T13:27:26,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T13:27:27,089 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data4/current/BP-1725618806-172.17.0.2-1732454845323/current, will proceed with Du for space computation calculation, 2024-11-24T13:27:27,089 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data1/current/BP-1725618806-172.17.0.2-1732454845323/current, will proceed with Du for space computation calculation, 2024-11-24T13:27:27,089 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data3/current/BP-1725618806-172.17.0.2-1732454845323/current, will proceed with Du for space computation calculation, 2024-11-24T13:27:27,089 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data2/current/BP-1725618806-172.17.0.2-1732454845323/current, will proceed with Du for space computation calculation, 2024-11-24T13:27:27,094 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/java.io.tmpdir/jetty-localhost-45907-hadoop-hdfs-3_4_1-tests_jar-_-any-4242329283321900721/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T13:27:27,095 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:45907} 2024-11-24T13:27:27,095 INFO [Time-limited test {}] server.Server(415): Started @3808ms 2024-11-24T13:27:27,097 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T13:27:27,148 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T13:27:27,148 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T13:27:27,219 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcbf9661173350c77 with lease ID 0xff48801a3a74647b: Processing first storage report for DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12 from datanode DatanodeRegistration(127.0.0.1:36111, datanodeUuid=5a7d5ab0-33c9-429d-ba0d-bfc5a68ff26d, infoPort=40655, infoSecurePort=0, ipcPort=40461, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323) 2024-11-24T13:27:27,220 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcbf9661173350c77 with lease ID 0xff48801a3a74647b: from storage DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12 node DatanodeRegistration(127.0.0.1:36111, datanodeUuid=5a7d5ab0-33c9-429d-ba0d-bfc5a68ff26d, infoPort=40655, infoSecurePort=0, ipcPort=40461, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T13:27:27,221 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdffd62e828dc465f with lease ID 0xff48801a3a74647c: Processing first storage report for DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe from datanode DatanodeRegistration(127.0.0.1:46321, datanodeUuid=5534b7c0-b2ba-4c2e-b79c-d95b1343c18b, infoPort=34299, infoSecurePort=0, ipcPort=40595, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323) 2024-11-24T13:27:27,221 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdffd62e828dc465f with lease ID 0xff48801a3a74647c: from storage DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe node DatanodeRegistration(127.0.0.1:46321, datanodeUuid=5534b7c0-b2ba-4c2e-b79c-d95b1343c18b, infoPort=34299, infoSecurePort=0, ipcPort=40595, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T13:27:27,221 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcbf9661173350c77 with lease ID 0xff48801a3a74647b: Processing first storage report for DS-5944379e-6e55-4ac0-a170-7f8c2752e82f from datanode DatanodeRegistration(127.0.0.1:36111, datanodeUuid=5a7d5ab0-33c9-429d-ba0d-bfc5a68ff26d, infoPort=40655, infoSecurePort=0, ipcPort=40461, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323) 2024-11-24T13:27:27,221 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcbf9661173350c77 with lease ID 0xff48801a3a74647b: from storage DS-5944379e-6e55-4ac0-a170-7f8c2752e82f node DatanodeRegistration(127.0.0.1:36111, datanodeUuid=5a7d5ab0-33c9-429d-ba0d-bfc5a68ff26d, infoPort=40655, infoSecurePort=0, ipcPort=40461, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T13:27:27,221 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdffd62e828dc465f with lease ID 0xff48801a3a74647c: Processing first storage report for DS-73d79936-12a0-48c1-8aaf-dafdca59db52 from datanode DatanodeRegistration(127.0.0.1:46321, datanodeUuid=5534b7c0-b2ba-4c2e-b79c-d95b1343c18b, infoPort=34299, infoSecurePort=0, ipcPort=40595, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323) 2024-11-24T13:27:27,221 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdffd62e828dc465f with lease ID 0xff48801a3a74647c: from storage DS-73d79936-12a0-48c1-8aaf-dafdca59db52 node DatanodeRegistration(127.0.0.1:46321, datanodeUuid=5534b7c0-b2ba-4c2e-b79c-d95b1343c18b, infoPort=34299, infoSecurePort=0, ipcPort=40595, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T13:27:27,239 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data6/current/BP-1725618806-172.17.0.2-1732454845323/current, will proceed with Du for space computation calculation, 2024-11-24T13:27:27,239 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data5/current/BP-1725618806-172.17.0.2-1732454845323/current, will proceed with Du for space computation calculation, 2024-11-24T13:27:27,271 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T13:27:27,277 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80aea66e65a2d95e with lease ID 0xff48801a3a74647d: Processing first storage report for DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a from datanode DatanodeRegistration(127.0.0.1:45659, datanodeUuid=4f5fc3d0-59f3-4f3f-a741-6ccc0cb7aad3, infoPort=38443, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323) 2024-11-24T13:27:27,277 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80aea66e65a2d95e with lease ID 0xff48801a3a74647d: from storage DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a node DatanodeRegistration(127.0.0.1:45659, datanodeUuid=4f5fc3d0-59f3-4f3f-a741-6ccc0cb7aad3, infoPort=38443, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T13:27:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80aea66e65a2d95e with lease ID 0xff48801a3a74647d: Processing first storage report for DS-52f46e76-65e6-4df8-bd99-8a166337f0bf from datanode DatanodeRegistration(127.0.0.1:45659, datanodeUuid=4f5fc3d0-59f3-4f3f-a741-6ccc0cb7aad3, infoPort=38443, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323) 2024-11-24T13:27:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80aea66e65a2d95e with lease ID 0xff48801a3a74647d: from storage DS-52f46e76-65e6-4df8-bd99-8a166337f0bf node DatanodeRegistration(127.0.0.1:45659, datanodeUuid=4f5fc3d0-59f3-4f3f-a741-6ccc0cb7aad3, infoPort=38443, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=328551435;c=1732454845323), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T13:27:27,476 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888 2024-11-24T13:27:27,558 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/zookeeper_0, clientPort=49808, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T13:27:27,570 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49808 2024-11-24T13:27:27,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:27,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:27,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741825_1001 (size=7) 2024-11-24T13:27:27,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741825_1001 (size=7) 2024-11-24T13:27:27,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741825_1001 (size=7) 2024-11-24T13:27:28,225 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07 with version=8 2024-11-24T13:27:28,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/hbase-staging 2024-11-24T13:27:28,527 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b10924491aa5:0 server-side Connection retries=45 2024-11-24T13:27:28,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:28,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:28,544 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:27:28,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:28,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:27:28,701 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T13:27:28,762 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T13:27:28,774 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T13:27:28,780 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:27:28,810 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20985 (auto-detected) 2024-11-24T13:27:28,811 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T13:27:28,831 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38765 2024-11-24T13:27:28,857 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38765 connecting to ZooKeeper ensemble=127.0.0.1:49808 2024-11-24T13:27:28,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387650x0, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:27:28,891 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38765-0x1014a28febd0000 connected 2024-11-24T13:27:28,916 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:28,919 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:28,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:27:28,932 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07, hbase.cluster.distributed=false 2024-11-24T13:27:28,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:27:28,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38765 2024-11-24T13:27:28,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38765 2024-11-24T13:27:28,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38765 2024-11-24T13:27:28,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38765 2024-11-24T13:27:28,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38765 2024-11-24T13:27:29,072 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b10924491aa5:0 server-side Connection retries=45 2024-11-24T13:27:29,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,074 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:27:29,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:27:29,077 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:27:29,079 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:27:29,080 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38813 2024-11-24T13:27:29,082 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38813 connecting to ZooKeeper ensemble=127.0.0.1:49808 2024-11-24T13:27:29,084 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,086 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388130x0, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:27:29,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38813-0x1014a28febd0001 connected 2024-11-24T13:27:29,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:27:29,098 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:27:29,106 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:27:29,108 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:27:29,114 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:27:29,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-24T13:27:29,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38813 2024-11-24T13:27:29,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38813 2024-11-24T13:27:29,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-24T13:27:29,117 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-24T13:27:29,134 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b10924491aa5:0 server-side Connection retries=45 2024-11-24T13:27:29,134 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,134 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,135 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:27:29,135 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,135 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:27:29,135 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:27:29,135 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:27:29,136 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34273 2024-11-24T13:27:29,138 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34273 connecting to ZooKeeper ensemble=127.0.0.1:49808 2024-11-24T13:27:29,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,141 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342730x0, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:27:29,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34273-0x1014a28febd0002 connected 2024-11-24T13:27:29,147 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:27:29,147 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:27:29,149 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:27:29,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:27:29,152 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:27:29,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34273 2024-11-24T13:27:29,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34273 2024-11-24T13:27:29,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34273 2024-11-24T13:27:29,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34273 2024-11-24T13:27:29,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34273 2024-11-24T13:27:29,178 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b10924491aa5:0 server-side Connection retries=45 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:27:29,179 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:27:29,180 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45225 2024-11-24T13:27:29,182 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45225 connecting to ZooKeeper ensemble=127.0.0.1:49808 2024-11-24T13:27:29,183 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,185 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452250x0, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:27:29,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452250x0, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:27:29,192 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45225-0x1014a28febd0003 connected 2024-11-24T13:27:29,192 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:27:29,193 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:27:29,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:27:29,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:27:29,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45225 2024-11-24T13:27:29,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45225 2024-11-24T13:27:29,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45225 2024-11-24T13:27:29,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45225 2024-11-24T13:27:29,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45225 2024-11-24T13:27:29,222 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b10924491aa5:38765 2024-11-24T13:27:29,223 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b10924491aa5,38765,1732454848333 2024-11-24T13:27:29,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,233 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b10924491aa5,38765,1732454848333 2024-11-24T13:27:29,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:27:29,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:27:29,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:27:29,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,259 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T13:27:29,261 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b10924491aa5,38765,1732454848333 from backup master directory 2024-11-24T13:27:29,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b10924491aa5,38765,1732454848333 2024-11-24T13:27:29,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:27:29,264 WARN [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:27:29,264 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b10924491aa5,38765,1732454848333 2024-11-24T13:27:29,267 INFO [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T13:27:29,268 INFO [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T13:27:29,325 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/hbase.id] with ID: 5f662c35-c445-40ae-a136-e68755f8700b 2024-11-24T13:27:29,325 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/.tmp/hbase.id 2024-11-24T13:27:29,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741826_1002 (size=42) 2024-11-24T13:27:29,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741826_1002 (size=42) 2024-11-24T13:27:29,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741826_1002 (size=42) 2024-11-24T13:27:29,341 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/.tmp/hbase.id]:[hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/hbase.id] 2024-11-24T13:27:29,386 INFO [master/b10924491aa5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T13:27:29,391 INFO [master/b10924491aa5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T13:27:29,409 INFO [master/b10924491aa5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T13:27:29,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741827_1003 (size=196) 2024-11-24T13:27:29,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741827_1003 (size=196) 2024-11-24T13:27:29,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741827_1003 (size=196) 2024-11-24T13:27:29,450 INFO [master/b10924491aa5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T13:27:29,453 INFO [master/b10924491aa5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T13:27:29,459 INFO [master/b10924491aa5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:29,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741828_1004 (size=1189) 2024-11-24T13:27:29,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741828_1004 (size=1189) 2024-11-24T13:27:29,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741828_1004 (size=1189) 2024-11-24T13:27:29,514 INFO [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store 2024-11-24T13:27:29,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741829_1005 (size=34) 2024-11-24T13:27:29,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741829_1005 (size=34) 2024-11-24T13:27:29,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741829_1005 (size=34) 2024-11-24T13:27:29,542 INFO [master/b10924491aa5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T13:27:29,545 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:29,547 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T13:27:29,547 INFO [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:27:29,547 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:27:29,549 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T13:27:29,549 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:27:29,550 INFO [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:27:29,551 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732454849546Disabling compacts and flushes for region at 1732454849546Disabling writes for close at 1732454849549 (+3 ms)Writing region close event to WAL at 1732454849550 (+1 ms)Closed at 1732454849550 2024-11-24T13:27:29,553 WARN [master/b10924491aa5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/.initializing 2024-11-24T13:27:29,553 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/WALs/b10924491aa5,38765,1732454848333 2024-11-24T13:27:29,561 INFO [master/b10924491aa5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:29,577 INFO [master/b10924491aa5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b10924491aa5%2C38765%2C1732454848333, suffix=, logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/WALs/b10924491aa5,38765,1732454848333, archiveDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/oldWALs, maxLogs=10 2024-11-24T13:27:29,617 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/WALs/b10924491aa5,38765,1732454848333/b10924491aa5%2C38765%2C1732454848333.1732454849583, exclude list is [], retry=0 2024-11-24T13:27:29,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:29,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:29,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:29,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:29,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T13:27:29,680 INFO [master/b10924491aa5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/WALs/b10924491aa5,38765,1732454848333/b10924491aa5%2C38765%2C1732454848333.1732454849583 2024-11-24T13:27:29,681 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:29,681 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:29,682 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:29,685 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,686 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T13:27:29,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:29,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:29,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,756 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T13:27:29,756 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:29,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:29,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T13:27:29,759 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:29,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:29,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T13:27:29,762 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:29,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:29,764 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,766 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,767 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,772 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,773 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,775 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:27:29,779 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:27:29,784 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:29,785 INFO [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74899150, jitterRate=0.11608430743217468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:27:29,790 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732454849699Initializing all the Stores at 1732454849701 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454849701Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454849702 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454849702Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454849702Cleaning up temporary data from old regions at 1732454849773 (+71 ms)Region opened successfully at 1732454849790 (+17 ms) 2024-11-24T13:27:29,791 INFO [master/b10924491aa5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T13:27:29,825 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d0d548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b10924491aa5/172.17.0.2:0 2024-11-24T13:27:29,857 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T13:27:29,869 INFO [master/b10924491aa5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T13:27:29,869 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T13:27:29,872 INFO [master/b10924491aa5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T13:27:29,873 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T13:27:29,878 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T13:27:29,879 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T13:27:29,912 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T13:27:29,922 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T13:27:29,924 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T13:27:29,926 INFO [master/b10924491aa5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T13:27:29,927 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T13:27:29,930 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T13:27:29,932 INFO [master/b10924491aa5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T13:27:29,935 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T13:27:29,937 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T13:27:29,938 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T13:27:29,939 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T13:27:29,956 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T13:27:29,958 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,966 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b10924491aa5,38765,1732454848333, sessionid=0x1014a28febd0000, setting cluster-up flag (Was=false) 2024-11-24T13:27:29,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,984 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T13:27:29,985 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b10924491aa5,38765,1732454848333 2024-11-24T13:27:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:29,996 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T13:27:29,997 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b10924491aa5,38765,1732454848333 2024-11-24T13:27:30,003 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T13:27:30,006 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(746): ClusterId : 5f662c35-c445-40ae-a136-e68755f8700b 2024-11-24T13:27:30,006 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(746): ClusterId : 5f662c35-c445-40ae-a136-e68755f8700b 2024-11-24T13:27:30,006 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(746): ClusterId : 5f662c35-c445-40ae-a136-e68755f8700b 2024-11-24T13:27:30,009 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:27:30,009 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:27:30,009 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:27:30,016 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:27:30,016 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:27:30,016 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:27:30,016 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:27:30,016 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:27:30,016 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:27:30,019 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:27:30,019 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:27:30,019 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:27:30,020 DEBUG [RS:2;b10924491aa5:45225 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c4b74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b10924491aa5/172.17.0.2:0 2024-11-24T13:27:30,020 DEBUG [RS:1;b10924491aa5:34273 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2826bae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b10924491aa5/172.17.0.2:0 2024-11-24T13:27:30,020 DEBUG [RS:0;b10924491aa5:38813 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1204cea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b10924491aa5/172.17.0.2:0 2024-11-24T13:27:30,035 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b10924491aa5:45225 2024-11-24T13:27:30,038 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b10924491aa5:34273 2024-11-24T13:27:30,038 INFO [RS:2;b10924491aa5:45225 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:27:30,038 INFO [RS:2;b10924491aa5:45225 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:27:30,038 INFO [RS:1;b10924491aa5:34273 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:27:30,038 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:27:30,038 INFO [RS:1;b10924491aa5:34273 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:27:30,038 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:27:30,041 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(2659): reportForDuty to master=b10924491aa5,38765,1732454848333 with port=45225, startcode=1732454849178 2024-11-24T13:27:30,041 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(2659): reportForDuty to master=b10924491aa5,38765,1732454848333 with port=34273, startcode=1732454849133 2024-11-24T13:27:30,045 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b10924491aa5:38813 2024-11-24T13:27:30,045 INFO [RS:0;b10924491aa5:38813 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:27:30,045 INFO [RS:0;b10924491aa5:38813 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:27:30,045 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:27:30,047 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(2659): reportForDuty to master=b10924491aa5,38765,1732454848333 with port=38813, startcode=1732454849031 2024-11-24T13:27:30,055 DEBUG [RS:2;b10924491aa5:45225 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:27:30,055 DEBUG [RS:0;b10924491aa5:38813 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:27:30,055 DEBUG [RS:1;b10924491aa5:34273 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:27:30,096 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:27:30,096 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:27:30,096 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:27:30,096 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T13:27:30,102 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T13:27:30,106 INFO [master/b10924491aa5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T13:27:30,109 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T13:27:30,110 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T13:27:30,113 INFO [master/b10924491aa5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T13:27:30,118 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b10924491aa5,38765,1732454848333 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T13:27:30,124 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b10924491aa5:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:27:30,124 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b10924491aa5:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:27:30,125 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b10924491aa5:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:27:30,125 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b10924491aa5:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:27:30,125 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b10924491aa5:0, corePoolSize=10, maxPoolSize=10 2024-11-24T13:27:30,125 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,125 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b10924491aa5:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:27:30,125 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,126 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732454880126 2024-11-24T13:27:30,128 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T13:27:30,129 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T13:27:30,131 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:27:30,131 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T13:27:30,131 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T13:27:30,131 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T13:27:30,131 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T13:27:30,131 WARN [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T13:27:30,131 WARN [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T13:27:30,131 WARN [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T13:27:30,132 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T13:27:30,133 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T13:27:30,133 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T13:27:30,133 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T13:27:30,134 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,136 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,137 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T13:27:30,137 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T13:27:30,138 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T13:27:30,138 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T13:27:30,140 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T13:27:30,140 INFO [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T13:27:30,143 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b10924491aa5:0:becomeActiveMaster-HFileCleaner.large.0-1732454850142,5,FailOnTimeoutGroup] 2024-11-24T13:27:30,144 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b10924491aa5:0:becomeActiveMaster-HFileCleaner.small.0-1732454850143,5,FailOnTimeoutGroup] 2024-11-24T13:27:30,144 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,144 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T13:27:30,145 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,146 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741831_1007 (size=1321) 2024-11-24T13:27:30,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741831_1007 (size=1321) 2024-11-24T13:27:30,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741831_1007 (size=1321) 2024-11-24T13:27:30,186 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T13:27:30,186 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07 2024-11-24T13:27:30,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741832_1008 (size=32) 2024-11-24T13:27:30,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741832_1008 (size=32) 2024-11-24T13:27:30,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741832_1008 (size=32) 2024-11-24T13:27:30,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:30,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T13:27:30,208 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:27:30,208 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T13:27:30,211 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:27:30,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T13:27:30,215 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:27:30,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T13:27:30,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:27:30,219 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:27:30,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740 2024-11-24T13:27:30,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740 2024-11-24T13:27:30,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:27:30,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:27:30,226 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:27:30,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:27:30,233 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(2659): reportForDuty to master=b10924491aa5,38765,1732454848333 with port=45225, startcode=1732454849178 2024-11-24T13:27:30,233 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(2659): reportForDuty to master=b10924491aa5,38765,1732454848333 with port=34273, startcode=1732454849133 2024-11-24T13:27:30,233 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(2659): reportForDuty to master=b10924491aa5,38765,1732454848333 with port=38813, startcode=1732454849031 2024-11-24T13:27:30,235 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:30,235 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b10924491aa5,38813,1732454849031 2024-11-24T13:27:30,236 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58773048, jitterRate=-0.1242133378982544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:27:30,238 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] master.ServerManager(517): Registering regionserver=b10924491aa5,38813,1732454849031 2024-11-24T13:27:30,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732454850203Initializing all the Stores at 1732454850205 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454850205Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454850205Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454850205Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454850205Cleaning up temporary data from old regions at 1732454850225 (+20 ms)Region opened successfully at 1732454850239 (+14 ms) 2024-11-24T13:27:30,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:27:30,240 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:27:30,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:27:30,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:27:30,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:27:30,241 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:27:30,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732454850239Disabling compacts and flushes for region at 1732454850239Disabling writes for close at 1732454850240 (+1 ms)Writing region close event to WAL at 1732454850241 (+1 ms)Closed at 1732454850241 2024-11-24T13:27:30,245 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:27:30,245 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T13:27:30,247 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b10924491aa5,45225,1732454849178 2024-11-24T13:27:30,247 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] master.ServerManager(517): Registering regionserver=b10924491aa5,45225,1732454849178 2024-11-24T13:27:30,247 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07 2024-11-24T13:27:30,247 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35337 2024-11-24T13:27:30,248 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:27:30,249 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,249 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38765 {}] master.ServerManager(517): Registering regionserver=b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,250 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07 2024-11-24T13:27:30,250 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35337 2024-11-24T13:27:30,250 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:27:30,252 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07 2024-11-24T13:27:30,252 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35337 2024-11-24T13:27:30,252 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:27:30,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T13:27:30,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:27:30,258 DEBUG [RS:0;b10924491aa5:38813 {}] zookeeper.ZKUtil(111): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b10924491aa5,38813,1732454849031 2024-11-24T13:27:30,258 DEBUG [RS:2;b10924491aa5:45225 {}] zookeeper.ZKUtil(111): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b10924491aa5,45225,1732454849178 2024-11-24T13:27:30,258 WARN [RS:0;b10924491aa5:38813 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:27:30,258 WARN [RS:2;b10924491aa5:45225 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:27:30,258 INFO [RS:2;b10924491aa5:45225 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:30,258 INFO [RS:0;b10924491aa5:38813 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:30,258 DEBUG [RS:1;b10924491aa5:34273 {}] zookeeper.ZKUtil(111): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,258 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,45225,1732454849178 2024-11-24T13:27:30,258 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031 2024-11-24T13:27:30,259 WARN [RS:1;b10924491aa5:34273 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:27:30,259 INFO [RS:1;b10924491aa5:34273 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:30,259 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,260 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b10924491aa5,38813,1732454849031] 2024-11-24T13:27:30,261 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b10924491aa5,34273,1732454849133] 2024-11-24T13:27:30,261 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b10924491aa5,45225,1732454849178] 2024-11-24T13:27:30,262 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:27:30,267 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T13:27:30,285 INFO [RS:2;b10924491aa5:45225 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:27:30,285 INFO [RS:0;b10924491aa5:38813 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:27:30,285 INFO [RS:1;b10924491aa5:34273 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:27:30,298 INFO [RS:2;b10924491aa5:45225 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:27:30,298 INFO [RS:0;b10924491aa5:38813 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:27:30,298 INFO [RS:1;b10924491aa5:34273 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:27:30,304 INFO [RS:2;b10924491aa5:45225 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:27:30,304 INFO [RS:1;b10924491aa5:34273 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:27:30,304 INFO [RS:0;b10924491aa5:38813 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:27:30,304 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,304 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,304 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,305 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:27:30,305 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:27:30,305 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:27:30,311 INFO [RS:0;b10924491aa5:38813 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:27:30,311 INFO [RS:1;b10924491aa5:34273 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:27:30,311 INFO [RS:2;b10924491aa5:45225 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:27:30,312 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,312 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,312 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,312 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,312 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,312 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,312 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,312 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,312 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,313 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b10924491aa5:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:27:30,314 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:2;b10924491aa5:45225 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b10924491aa5:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:27:30,314 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b10924491aa5:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:27:30,314 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,314 DEBUG [RS:0;b10924491aa5:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b10924491aa5:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:27:30,314 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b10924491aa5:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:27:30,315 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b10924491aa5:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:27:30,315 DEBUG [RS:1;b10924491aa5:34273 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b10924491aa5:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:27:30,315 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,315 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,315 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,316 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,316 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,316 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,45225,1732454849178-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,34273,1732454849133-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,317 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38813,1732454849031-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:27:30,338 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:27:30,340 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38813,1732454849031-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,340 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,340 INFO [RS:0;b10924491aa5:38813 {}] regionserver.Replication(171): b10924491aa5,38813,1732454849031 started 2024-11-24T13:27:30,341 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:27:30,342 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,45225,1732454849178-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,342 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:27:30,342 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,342 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,34273,1732454849133-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,342 INFO [RS:2;b10924491aa5:45225 {}] regionserver.Replication(171): b10924491aa5,45225,1732454849178 started 2024-11-24T13:27:30,342 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,342 INFO [RS:1;b10924491aa5:34273 {}] regionserver.Replication(171): b10924491aa5,34273,1732454849133 started 2024-11-24T13:27:30,359 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,359 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1482): Serving as b10924491aa5,38813,1732454849031, RpcServer on b10924491aa5/172.17.0.2:38813, sessionid=0x1014a28febd0001 2024-11-24T13:27:30,360 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:27:30,360 DEBUG [RS:0;b10924491aa5:38813 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b10924491aa5,38813,1732454849031 2024-11-24T13:27:30,360 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b10924491aa5,38813,1732454849031' 2024-11-24T13:27:30,360 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:27:30,361 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:27:30,361 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,361 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1482): Serving as b10924491aa5,45225,1732454849178, RpcServer on b10924491aa5/172.17.0.2:45225, sessionid=0x1014a28febd0003 2024-11-24T13:27:30,362 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:27:30,362 DEBUG [RS:2;b10924491aa5:45225 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b10924491aa5,45225,1732454849178 2024-11-24T13:27:30,362 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b10924491aa5,45225,1732454849178' 2024-11-24T13:27:30,362 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:27:30,362 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:27:30,362 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:27:30,362 DEBUG [RS:0;b10924491aa5:38813 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b10924491aa5,38813,1732454849031 2024-11-24T13:27:30,362 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b10924491aa5,38813,1732454849031' 2024-11-24T13:27:30,363 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:27:30,363 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:27:30,363 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:27:30,363 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:27:30,363 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:27:30,363 DEBUG [RS:2;b10924491aa5:45225 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b10924491aa5,45225,1732454849178 2024-11-24T13:27:30,363 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b10924491aa5,45225,1732454849178' 2024-11-24T13:27:30,363 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:27:30,364 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:27:30,364 DEBUG [RS:0;b10924491aa5:38813 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:27:30,364 INFO [RS:0;b10924491aa5:38813 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:27:30,364 INFO [RS:0;b10924491aa5:38813 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:27:30,365 DEBUG [RS:2;b10924491aa5:45225 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:27:30,365 INFO [RS:2;b10924491aa5:45225 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:27:30,365 INFO [RS:2;b10924491aa5:45225 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:27:30,365 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:30,366 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1482): Serving as b10924491aa5,34273,1732454849133, RpcServer on b10924491aa5/172.17.0.2:34273, sessionid=0x1014a28febd0002 2024-11-24T13:27:30,366 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:27:30,366 DEBUG [RS:1;b10924491aa5:34273 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,366 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b10924491aa5,34273,1732454849133' 2024-11-24T13:27:30,366 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:27:30,367 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:27:30,367 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:27:30,367 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:27:30,368 DEBUG [RS:1;b10924491aa5:34273 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,368 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b10924491aa5,34273,1732454849133' 2024-11-24T13:27:30,368 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:27:30,368 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:27:30,369 DEBUG [RS:1;b10924491aa5:34273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:27:30,369 INFO [RS:1;b10924491aa5:34273 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:27:30,369 INFO [RS:1;b10924491aa5:34273 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:27:30,418 WARN [b10924491aa5:38765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T13:27:30,469 INFO [RS:2;b10924491aa5:45225 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:30,469 INFO [RS:0;b10924491aa5:38813 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:30,470 INFO [RS:1;b10924491aa5:34273 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:30,473 INFO [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b10924491aa5%2C38813%2C1732454849031, suffix=, logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031, archiveDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs, maxLogs=32 2024-11-24T13:27:30,473 INFO [RS:2;b10924491aa5:45225 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b10924491aa5%2C45225%2C1732454849178, suffix=, logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,45225,1732454849178, archiveDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs, maxLogs=32 2024-11-24T13:27:30,473 INFO [RS:1;b10924491aa5:34273 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b10924491aa5%2C34273%2C1732454849133, suffix=, logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133, archiveDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs, maxLogs=32 2024-11-24T13:27:30,490 DEBUG [RS:2;b10924491aa5:45225 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,45225,1732454849178/b10924491aa5%2C45225%2C1732454849178.1732454850477, exclude list is [], retry=0 2024-11-24T13:27:30,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:30,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:30,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:30,498 DEBUG [RS:0;b10924491aa5:38813 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031/b10924491aa5%2C38813%2C1732454849031.1732454850477, exclude list is [], retry=0 2024-11-24T13:27:30,498 DEBUG [RS:1;b10924491aa5:34273 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133/b10924491aa5%2C34273%2C1732454849133.1732454850477, exclude list is [], retry=0 2024-11-24T13:27:30,500 INFO [RS:2;b10924491aa5:45225 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,45225,1732454849178/b10924491aa5%2C45225%2C1732454849178.1732454850477 2024-11-24T13:27:30,503 DEBUG [RS:2;b10924491aa5:45225 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:30,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:30,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:30,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:30,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:30,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:30,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:30,514 INFO [RS:1;b10924491aa5:34273 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133/b10924491aa5%2C34273%2C1732454849133.1732454850477 2024-11-24T13:27:30,514 INFO [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031/b10924491aa5%2C38813%2C1732454849031.1732454850477 2024-11-24T13:27:30,514 DEBUG [RS:1;b10924491aa5:34273 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:30,514 DEBUG [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:30,671 DEBUG [b10924491aa5:38765 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T13:27:30,678 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(204): Hosts are {b10924491aa5=0} racks are {/default-rack=0} 2024-11-24T13:27:30,684 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:27:30,684 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:27:30,684 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T13:27:30,684 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:27:30,684 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:27:30,684 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T13:27:30,685 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:27:30,685 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:27:30,685 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T13:27:30,685 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:27:30,691 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,698 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b10924491aa5,34273,1732454849133, state=OPENING 2024-11-24T13:27:30,703 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T13:27:30,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:30,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:30,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:30,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:27:30,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,707 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:27:30,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b10924491aa5,34273,1732454849133}] 2024-11-24T13:27:30,883 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:27:30,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48823, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:27:30,897 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T13:27:30,898 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:30,898 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T13:27:30,901 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b10924491aa5%2C34273%2C1732454849133.meta, suffix=.meta, logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133, archiveDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs, maxLogs=32 2024-11-24T13:27:30,916 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, exclude list is [], retry=0 2024-11-24T13:27:30,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:30,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:30,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:30,923 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta 2024-11-24T13:27:30,924 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:30,924 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:30,926 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T13:27:30,929 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T13:27:30,934 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T13:27:30,939 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T13:27:30,940 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:30,940 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T13:27:30,940 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T13:27:30,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T13:27:30,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:27:30,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T13:27:30,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:27:30,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T13:27:30,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:27:30,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T13:27:30,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:27:30,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:30,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:30,953 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:27:30,955 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740 2024-11-24T13:27:30,957 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740 2024-11-24T13:27:30,959 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:27:30,959 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:27:30,960 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:27:30,963 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:27:30,965 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71587143, jitterRate=0.06673155725002289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:27:30,965 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T13:27:30,967 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732454850942Writing region info on filesystem at 1732454850942Initializing all the Stores at 1732454850944 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454850944Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454850944Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454850944Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454850945 (+1 ms)Cleaning up temporary data from old regions at 1732454850959 (+14 ms)Running coprocessor post-open hooks at 1732454850965 (+6 ms)Region opened successfully at 1732454850966 (+1 ms) 2024-11-24T13:27:30,973 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732454850874 2024-11-24T13:27:30,984 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T13:27:30,985 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T13:27:30,986 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,989 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b10924491aa5,34273,1732454849133, state=OPEN 2024-11-24T13:27:30,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:30,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:30,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:30,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:30,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:30,992 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b10924491aa5,34273,1732454849133 2024-11-24T13:27:30,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T13:27:30,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b10924491aa5,34273,1732454849133 in 284 msec 2024-11-24T13:27:31,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T13:27:31,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 747 msec 2024-11-24T13:27:31,005 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:27:31,005 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T13:27:31,025 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:31,026 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1] 2024-11-24T13:27:31,047 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:27:31,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54543, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:27:31,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0300 sec 2024-11-24T13:27:31,068 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732454851068, completionTime=-1 2024-11-24T13:27:31,071 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T13:27:31,071 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T13:27:31,099 INFO [master/b10924491aa5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T13:27:31,099 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732454911099 2024-11-24T13:27:31,099 INFO [master/b10924491aa5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732454971099 2024-11-24T13:27:31,100 INFO [master/b10924491aa5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-24T13:27:31,101 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T13:27:31,110 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38765,1732454848333-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,110 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38765,1732454848333-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,110 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38765,1732454848333-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,112 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b10924491aa5:38765, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,113 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,113 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,119 DEBUG [master/b10924491aa5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T13:27:31,141 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.876sec 2024-11-24T13:27:31,142 INFO [master/b10924491aa5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T13:27:31,143 INFO [master/b10924491aa5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T13:27:31,144 INFO [master/b10924491aa5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T13:27:31,145 INFO [master/b10924491aa5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T13:27:31,145 INFO [master/b10924491aa5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T13:27:31,146 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38765,1732454848333-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:27:31,146 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38765,1732454848333-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T13:27:31,151 DEBUG [master/b10924491aa5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T13:27:31,152 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T13:27:31,152 INFO [master/b10924491aa5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b10924491aa5,38765,1732454848333-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:31,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e28cab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:27:31,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b10924491aa5,38765,-1 for getting cluster id 2024-11-24T13:27:31,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T13:27:31,231 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5f662c35-c445-40ae-a136-e68755f8700b' 2024-11-24T13:27:31,233 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T13:27:31,234 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5f662c35-c445-40ae-a136-e68755f8700b" 2024-11-24T13:27:31,234 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411c0491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:27:31,234 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b10924491aa5,38765,-1] 2024-11-24T13:27:31,238 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T13:27:31,240 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:27:31,241 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T13:27:31,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b58e27e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:27:31,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:31,251 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1] 2024-11-24T13:27:31,252 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:27:31,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:27:31,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b10924491aa5,38765,1732454848333 2024-11-24T13:27:31,276 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35337/hbase 2024-11-24T13:27:31,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=167, ProcessCount=11, AvailableMemoryMB=7585 2024-11-24T13:27:31,311 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:31,314 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:31,315 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:31,321 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97920154, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-97920154, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:31,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97920154/hregion-97920154.1732454851323, exclude list is [], retry=0 2024-11-24T13:27:31,349 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:31,349 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:31,350 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:31,354 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97920154/hregion-97920154.1732454851323 2024-11-24T13:27:31,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:31,355 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e404ea4120a79a4b035a30aa0388fd72, NAME => 'testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:31,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741838_1014 (size=64) 2024-11-24T13:27:31,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741838_1014 (size=64) 2024-11-24T13:27:31,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741838_1014 (size=64) 2024-11-24T13:27:31,368 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:31,370 INFO [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,372 INFO [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e404ea4120a79a4b035a30aa0388fd72 columnFamilyName a 2024-11-24T13:27:31,372 DEBUG [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:31,373 INFO [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] regionserver.HStore(327): Store=e404ea4120a79a4b035a30aa0388fd72/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:31,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,376 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,376 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,379 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,383 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:31,383 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e404ea4120a79a4b035a30aa0388fd72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68966980, jitterRate=0.02768808603286743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:31,385 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e404ea4120a79a4b035a30aa0388fd72: Writing region info on filesystem at 1732454851368Initializing all the Stores at 1732454851370 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454851370Cleaning up temporary data from old regions at 1732454851376 (+6 ms)Region opened successfully at 1732454851385 (+9 ms) 2024-11-24T13:27:31,385 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e404ea4120a79a4b035a30aa0388fd72, disabling compactions & flushes 2024-11-24T13:27:31,385 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72. 2024-11-24T13:27:31,385 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72. 2024-11-24T13:27:31,385 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72. after waiting 0 ms 2024-11-24T13:27:31,385 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72. 2024-11-24T13:27:31,386 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72. 2024-11-24T13:27:31,386 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e404ea4120a79a4b035a30aa0388fd72: Waiting for close lock at 1732454851385Disabling compacts and flushes for region at 1732454851385Disabling writes for close at 1732454851385Writing region close event to WAL at 1732454851386 (+1 ms)Closed at 1732454851386 2024-11-24T13:27:31,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741837_1013 (size=93) 2024-11-24T13:27:31,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741837_1013 (size=93) 2024-11-24T13:27:31,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741837_1013 (size=93) 2024-11-24T13:27:31,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:31,397 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97920154:(num 1732454851323) 2024-11-24T13:27:31,399 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T13:27:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741839_1015 (size=369) 2024-11-24T13:27:31,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741839_1015 (size=369) 2024-11-24T13:27:31,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741839_1015 (size=369) 2024-11-24T13:27:31,421 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T13:27:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741840_1016 (size=231) 2024-11-24T13:27:31,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741840_1016 (size=231) 2024-11-24T13:27:31,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741840_1016 (size=231) 2024-11-24T13:27:31,456 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1, size=369 (369bytes) 2024-11-24T13:27:31,457 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T13:27:31,457 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T13:27:31,457 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1 2024-11-24T13:27:31,463 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1 after 4ms 2024-11-24T13:27:31,470 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1 took 15ms 2024-11-24T13:27:31,474 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1 so closing down 2024-11-24T13:27:31,477 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T13:27:31,479 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T13:27:31,479 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:31,480 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:31,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741841_1017 (size=369) 2024-11-24T13:27:31,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741841_1017 (size=369) 2024-11-24T13:27:31,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741841_1017 (size=369) 2024-11-24T13:27:31,495 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:31,497 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002 2024-11-24T13:27:31,502 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-24T13:27:31,503 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1, journal: Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1, size=369 (369bytes) at 1732454851456Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1 so closing down at 1732454851474 (+18 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000001-wal-1.temp at 1732454851479 (+5 ms)3 split writer threads finished at 1732454851480 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732454851495 (+15 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002 at 1732454851498 (+3 ms)Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732454851502 (+4 ms) 2024-11-24T13:27:31,528 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2, size=231 (231bytes) 2024-11-24T13:27:31,528 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2 2024-11-24T13:27:31,529 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2 after 1ms 2024-11-24T13:27:31,533 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2 took 6ms 2024-11-24T13:27:31,536 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2 so closing down 2024-11-24T13:27:31,536 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:31,539 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T13:27:31,541 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T13:27:31,541 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741842_1018 (size=231) 2024-11-24T13:27:31,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741842_1018 (size=231) 2024-11-24T13:27:31,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741842_1018 (size=231) 2024-11-24T13:27:31,551 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:31,557 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-24T13:27:31,559 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-24T13:27:31,559 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2, journal: Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2, size=231 (231bytes) at 1732454851528Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2 so closing down at 1732454851536 (+8 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002-wal-2.temp at 1732454851541 (+5 ms)3 split writer threads finished at 1732454851541Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732454851551 (+10 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732454851559 (+8 ms) 2024-11-24T13:27:31,560 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:31,562 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:31,586 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal.1732454851564, exclude list is [], retry=0 2024-11-24T13:27:31,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:31,591 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:31,591 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:31,594 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal.1732454851564 2024-11-24T13:27:31,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:31,594 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e404ea4120a79a4b035a30aa0388fd72, NAME => 'testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:31,594 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:31,595 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,595 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,597 INFO [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,599 INFO [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e404ea4120a79a4b035a30aa0388fd72 columnFamilyName a 2024-11-24T13:27:31,599 DEBUG [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:31,600 INFO [StoreOpener-e404ea4120a79a4b035a30aa0388fd72-1 {}] regionserver.HStore(327): Store=e404ea4120a79a4b035a30aa0388fd72/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:31,600 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,601 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,605 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,607 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002 2024-11-24T13:27:31,617 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002 2024-11-24T13:27:31,622 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e404ea4120a79a4b035a30aa0388fd72 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T13:27:31,683 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/.tmp/a/e4bb14cdb9814829bb04616cca5a0f23 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732454851397/Put/seqid=0 2024-11-24T13:27:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741844_1020 (size=5170) 2024-11-24T13:27:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741844_1020 (size=5170) 2024-11-24T13:27:31,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741844_1020 (size=5170) 2024-11-24T13:27:31,699 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/.tmp/a/e4bb14cdb9814829bb04616cca5a0f23 2024-11-24T13:27:31,775 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/.tmp/a/e4bb14cdb9814829bb04616cca5a0f23 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/a/e4bb14cdb9814829bb04616cca5a0f23 2024-11-24T13:27:31,785 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/a/e4bb14cdb9814829bb04616cca5a0f23, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T13:27:31,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e404ea4120a79a4b035a30aa0388fd72 in 170ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T13:27:31,794 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/0000000000000000002 2024-11-24T13:27:31,795 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,795 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,798 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e404ea4120a79a4b035a30aa0388fd72 2024-11-24T13:27:31,801 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/e404ea4120a79a4b035a30aa0388fd72/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T13:27:31,802 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e404ea4120a79a4b035a30aa0388fd72; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66977720, jitterRate=-0.001954197883605957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:31,804 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e404ea4120a79a4b035a30aa0388fd72: Writing region info on filesystem at 1732454851595Initializing all the Stores at 1732454851597 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454851597Obtaining lock to block concurrent updates at 1732454851622 (+25 ms)Preparing flush snapshotting stores in e404ea4120a79a4b035a30aa0388fd72 at 1732454851622Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732454851626 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732454851312.e404ea4120a79a4b035a30aa0388fd72. at 1732454851626Flushing e404ea4120a79a4b035a30aa0388fd72/a: creating writer at 1732454851628 (+2 ms)Flushing e404ea4120a79a4b035a30aa0388fd72/a: appending metadata at 1732454851673 (+45 ms)Flushing e404ea4120a79a4b035a30aa0388fd72/a: closing flushed file at 1732454851676 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d3e36bd: reopening flushed file at 1732454851772 (+96 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e404ea4120a79a4b035a30aa0388fd72 in 170ms, sequenceid=2, compaction requested=false; wal=null at 1732454851792 (+20 ms)Cleaning up temporary data from old regions at 1732454851795 (+3 ms)Region opened successfully at 1732454851803 (+8 ms) 2024-11-24T13:27:31,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=378 (was 366) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:45008 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53178 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:47292 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:50896 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35337/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53154 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:59582 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=167 (was 167), ProcessCount=11 (was 11), AvailableMemoryMB=7490 (was 7585) 2024-11-24T13:27:31,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=167, ProcessCount=11, AvailableMemoryMB=7489 2024-11-24T13:27:31,866 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:31,868 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:31,869 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:31,873 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70687227, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-70687227, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:31,887 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70687227/hregion-70687227.1732454851873, exclude list is [], retry=0 2024-11-24T13:27:31,891 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:31,891 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:31,891 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:31,895 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70687227/hregion-70687227.1732454851873 2024-11-24T13:27:31,895 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:31,895 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 46574c0dca95750326eed9cf474c08dc, NAME => 'testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:31,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741846_1022 (size=64) 2024-11-24T13:27:31,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741846_1022 (size=64) 2024-11-24T13:27:31,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741846_1022 (size=64) 2024-11-24T13:27:32,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:32,313 INFO [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,315 INFO [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46574c0dca95750326eed9cf474c08dc columnFamilyName a 2024-11-24T13:27:32,315 DEBUG [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:32,316 INFO [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] regionserver.HStore(327): Store=46574c0dca95750326eed9cf474c08dc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:32,316 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,318 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,318 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,318 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,321 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,324 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:32,325 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 46574c0dca95750326eed9cf474c08dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60939198, jitterRate=-0.0919351875782013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 46574c0dca95750326eed9cf474c08dc: Writing region info on filesystem at 1732454852311Initializing all the Stores at 1732454852313 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454852313Cleaning up temporary data from old regions at 1732454852318 (+5 ms)Region opened successfully at 1732454852325 (+7 ms) 2024-11-24T13:27:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 46574c0dca95750326eed9cf474c08dc, disabling compactions & flushes 2024-11-24T13:27:32,325 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc. 2024-11-24T13:27:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc. 2024-11-24T13:27:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc. after waiting 0 ms 2024-11-24T13:27:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc. 2024-11-24T13:27:32,325 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc. 2024-11-24T13:27:32,325 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 46574c0dca95750326eed9cf474c08dc: Waiting for close lock at 1732454852325Disabling compacts and flushes for region at 1732454852325Disabling writes for close at 1732454852325Writing region close event to WAL at 1732454852325Closed at 1732454852325 2024-11-24T13:27:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741845_1021 (size=93) 2024-11-24T13:27:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741845_1021 (size=93) 2024-11-24T13:27:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741845_1021 (size=93) 2024-11-24T13:27:32,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:32,332 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70687227:(num 1732454851873) 2024-11-24T13:27:32,333 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T13:27:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741847_1023 (size=369) 2024-11-24T13:27:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741847_1023 (size=369) 2024-11-24T13:27:32,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741847_1023 (size=369) 2024-11-24T13:27:32,346 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T13:27:32,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741848_1024 (size=231) 2024-11-24T13:27:32,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741848_1024 (size=231) 2024-11-24T13:27:32,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741848_1024 (size=231) 2024-11-24T13:27:32,372 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2, size=231 (231bytes) 2024-11-24T13:27:32,372 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2 2024-11-24T13:27:32,373 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2 after 0ms 2024-11-24T13:27:32,377 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2 took 5ms 2024-11-24T13:27:32,379 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2 so closing down 2024-11-24T13:27:32,379 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:32,381 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T13:27:32,383 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T13:27:32,383 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:32,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741849_1025 (size=231) 2024-11-24T13:27:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741849_1025 (size=231) 2024-11-24T13:27:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741849_1025 (size=231) 2024-11-24T13:27:32,391 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:32,393 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 2024-11-24T13:27:32,393 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-24T13:27:32,393 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2, journal: Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2, size=231 (231bytes) at 1732454852372Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2 so closing down at 1732454852379 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002-wal-2.temp at 1732454852383 (+4 ms)3 split writer threads finished at 1732454852383Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732454852391 (+8 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 at 1732454852393 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732454852393 2024-11-24T13:27:32,408 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1, size=369 (369bytes) 2024-11-24T13:27:32,408 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1 2024-11-24T13:27:32,409 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1 after 1ms 2024-11-24T13:27:32,412 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1 took 4ms 2024-11-24T13:27:32,414 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1 so closing down 2024-11-24T13:27:32,414 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:32,416 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T13:27:32,417 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T13:27:32,417 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741850_1026 (size=369) 2024-11-24T13:27:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741850_1026 (size=369) 2024-11-24T13:27:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741850_1026 (size=369) 2024-11-24T13:27:32,425 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:32,431 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002, length=231 2024-11-24T13:27:32,433 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 2024-11-24T13:27:32,433 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-24T13:27:32,433 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1, journal: Splitting hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1, size=369 (369bytes) at 1732454852408Finishing writing output for hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1 so closing down at 1732454852414 (+6 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000001-wal-1.temp at 1732454852417 (+3 ms)3 split writer threads finished at 1732454852417Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732454852425 (+8 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 at 1732454852433 (+8 ms)Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732454852433 2024-11-24T13:27:32,433 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:32,435 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:32,451 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal.1732454852436, exclude list is [], retry=0 2024-11-24T13:27:32,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:32,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:32,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:32,457 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal.1732454852436 2024-11-24T13:27:32,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:32,457 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 46574c0dca95750326eed9cf474c08dc, NAME => 'testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:32,458 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:32,458 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,458 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,460 INFO [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,461 INFO [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46574c0dca95750326eed9cf474c08dc columnFamilyName a 2024-11-24T13:27:32,462 DEBUG [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:32,462 INFO [StoreOpener-46574c0dca95750326eed9cf474c08dc-1 {}] regionserver.HStore(327): Store=46574c0dca95750326eed9cf474c08dc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:32,462 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,466 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 2024-11-24T13:27:32,471 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 2024-11-24T13:27:32,471 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 46574c0dca95750326eed9cf474c08dc 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T13:27:32,487 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/.tmp/a/7d2355a6382d4ce3af0f252feb6a9ed7 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732454852332/Put/seqid=0 2024-11-24T13:27:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741852_1028 (size=5170) 2024-11-24T13:27:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741852_1028 (size=5170) 2024-11-24T13:27:32,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741852_1028 (size=5170) 2024-11-24T13:27:32,496 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/.tmp/a/7d2355a6382d4ce3af0f252feb6a9ed7 2024-11-24T13:27:32,504 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/.tmp/a/7d2355a6382d4ce3af0f252feb6a9ed7 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/a/7d2355a6382d4ce3af0f252feb6a9ed7 2024-11-24T13:27:32,512 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/a/7d2355a6382d4ce3af0f252feb6a9ed7, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T13:27:32,512 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 46574c0dca95750326eed9cf474c08dc in 41ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T13:27:32,513 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/0000000000000000002 2024-11-24T13:27:32,513 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,513 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,516 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 46574c0dca95750326eed9cf474c08dc 2024-11-24T13:27:32,518 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/46574c0dca95750326eed9cf474c08dc/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T13:27:32,519 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 46574c0dca95750326eed9cf474c08dc; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64617835, jitterRate=-0.03711922466754913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:32,519 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 46574c0dca95750326eed9cf474c08dc: Writing region info on filesystem at 1732454852458Initializing all the Stores at 1732454852459 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454852459Obtaining lock to block concurrent updates at 1732454852471 (+12 ms)Preparing flush snapshotting stores in 46574c0dca95750326eed9cf474c08dc at 1732454852471Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732454852471Flushing stores of testReplayEditsWrittenIntoWAL,,1732454851867.46574c0dca95750326eed9cf474c08dc. at 1732454852471Flushing 46574c0dca95750326eed9cf474c08dc/a: creating writer at 1732454852471Flushing 46574c0dca95750326eed9cf474c08dc/a: appending metadata at 1732454852486 (+15 ms)Flushing 46574c0dca95750326eed9cf474c08dc/a: closing flushed file at 1732454852486Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14dd0afc: reopening flushed file at 1732454852503 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 46574c0dca95750326eed9cf474c08dc in 41ms, sequenceid=2, compaction requested=false; wal=null at 1732454852512 (+9 ms)Cleaning up temporary data from old regions at 1732454852513 (+1 ms)Region opened successfully at 1732454852519 (+6 ms) 2024-11-24T13:27:32,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:47362 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53260 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53154 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:59582 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:50966 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=167 (was 167), ProcessCount=11 (was 11), AvailableMemoryMB=7484 (was 7489) 2024-11-24T13:27:32,548 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=167, ProcessCount=11, AvailableMemoryMB=7483 2024-11-24T13:27:32,563 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:32,565 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:32,566 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:32,569 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36718087, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-36718087, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:32,582 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36718087/hregion-36718087.1732454852569, exclude list is [], retry=0 2024-11-24T13:27:32,585 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:32,586 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:32,586 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:32,588 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36718087/hregion-36718087.1732454852569 2024-11-24T13:27:32,589 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:32,589 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 338df40f90680e5389f786b66d651d31, NAME => 'testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741854_1030 (size=64) 2024-11-24T13:27:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741854_1030 (size=64) 2024-11-24T13:27:32,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741854_1030 (size=64) 2024-11-24T13:27:32,599 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:32,601 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,603 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338df40f90680e5389f786b66d651d31 columnFamilyName a 2024-11-24T13:27:32,603 DEBUG [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:32,604 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(327): Store=338df40f90680e5389f786b66d651d31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:32,604 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,605 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338df40f90680e5389f786b66d651d31 columnFamilyName b 2024-11-24T13:27:32,605 DEBUG [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:32,606 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(327): Store=338df40f90680e5389f786b66d651d31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:32,606 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,608 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338df40f90680e5389f786b66d651d31 columnFamilyName c 2024-11-24T13:27:32,608 DEBUG [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:32,608 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(327): Store=338df40f90680e5389f786b66d651d31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:32,608 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,609 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,610 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,611 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,611 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,612 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:32,613 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:32,616 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:32,616 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 338df40f90680e5389f786b66d651d31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71328831, jitterRate=0.06288240849971771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:32,617 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 338df40f90680e5389f786b66d651d31: Writing region info on filesystem at 1732454852600Initializing all the Stores at 1732454852601 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454852601Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454852601Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454852601Cleaning up temporary data from old regions at 1732454852611 (+10 ms)Region opened successfully at 1732454852617 (+6 ms) 2024-11-24T13:27:32,617 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 338df40f90680e5389f786b66d651d31, disabling compactions & flushes 2024-11-24T13:27:32,617 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:32,617 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:32,617 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. after waiting 0 ms 2024-11-24T13:27:32,617 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:32,617 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:32,618 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 338df40f90680e5389f786b66d651d31: Waiting for close lock at 1732454852617Disabling compacts and flushes for region at 1732454852617Disabling writes for close at 1732454852617Writing region close event to WAL at 1732454852617Closed at 1732454852617 2024-11-24T13:27:32,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741853_1029 (size=93) 2024-11-24T13:27:32,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741853_1029 (size=93) 2024-11-24T13:27:32,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741853_1029 (size=93) 2024-11-24T13:27:32,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:32,625 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36718087:(num 1732454852569) 2024-11-24T13:27:32,625 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:32,627 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:32,642 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628, exclude list is [], retry=0 2024-11-24T13:27:32,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:32,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:32,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:32,649 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 2024-11-24T13:27:32,649 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:32,821 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628, size=0 (0bytes) 2024-11-24T13:27:32,821 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 might be still open, length is 0 2024-11-24T13:27:32,821 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 2024-11-24T13:27:32,822 WARN [IPC Server handler 3 on default port 35337 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T13:27:32,823 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 after 2ms 2024-11-24T13:27:33,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:51008 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36111:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51008 dst: /127.0.0.1:36111 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36111 remote=/127.0.0.1:51008]. Total timeout mills is 60000, 59495 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:33,294 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:47394 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47394 dst: /127.0.0.1:45659 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:33,294 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53302 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53302 dst: /127.0.0.1:46321 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:33,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741855_1032 (size=470675) 2024-11-24T13:27:33,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741855_1032 (size=470675) 2024-11-24T13:27:33,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741855_1032 (size=470675) 2024-11-24T13:27:36,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T13:27:36,491 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T13:27:36,824 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 after 4002ms 2024-11-24T13:27:36,829 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 took 4008ms 2024-11-24T13:27:36,834 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732454852628.temp 2024-11-24T13:27:36,840 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000000001-wal.1732454852628.temp 2024-11-24T13:27:36,905 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628; continuing. 2024-11-24T13:27:36,905 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 so closing down 2024-11-24T13:27:36,905 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:36,905 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:36,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741856_1033 (size=470683) 2024-11-24T13:27:36,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741856_1033 (size=470683) 2024-11-24T13:27:36,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741856_1033 (size=470683) 2024-11-24T13:27:36,910 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000000001-wal.1732454852628.temp (wrote 3002 edits, skipped 0 edits in 27 ms) 2024-11-24T13:27:36,912 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000000001-wal.1732454852628.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000003002 2024-11-24T13:27:36,912 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 83 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T13:27:36,913 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628, journal: Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628, size=0 (0bytes) at 1732454852821Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000000001-wal.1732454852628.temp at 1732454856840 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1732454856879 (+39 ms)Split 2048 edits, skipped 0 edits. at 1732454856893 (+14 ms)Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 so closing down at 1732454856905 (+12 ms)3 split writer threads finished at 1732454856905Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000000001-wal.1732454852628.temp (wrote 3002 edits, skipped 0 edits in 27 ms) at 1732454856910 (+5 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000000001-wal.1732454852628.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000003002 at 1732454856912 (+2 ms)Processed 3002 edits across 1 Regions in 83 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628, size=0, length=0, corrupted=false, cancelled=false at 1732454856912 2024-11-24T13:27:36,915 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454852628 2024-11-24T13:27:36,916 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000003002 2024-11-24T13:27:36,917 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:36,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:36,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454856925, exclude list is [], retry=0 2024-11-24T13:27:36,945 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:36,945 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:36,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:36,948 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454856925 2024-11-24T13:27:36,948 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:36,949 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:36,951 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:36,952 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338df40f90680e5389f786b66d651d31 columnFamilyName a 2024-11-24T13:27:36,952 DEBUG [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:36,953 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(327): Store=338df40f90680e5389f786b66d651d31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:36,953 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:36,954 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338df40f90680e5389f786b66d651d31 columnFamilyName b 2024-11-24T13:27:36,955 DEBUG [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:36,955 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(327): Store=338df40f90680e5389f786b66d651d31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:36,955 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:36,956 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338df40f90680e5389f786b66d651d31 columnFamilyName c 2024-11-24T13:27:36,956 DEBUG [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:36,957 INFO [StoreOpener-338df40f90680e5389f786b66d651d31-1 {}] regionserver.HStore(327): Store=338df40f90680e5389f786b66d651d31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:36,957 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:36,958 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31 2024-11-24T13:27:36,960 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31 2024-11-24T13:27:36,961 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000003002 2024-11-24T13:27:36,993 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T13:27:37,361 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 338df40f90680e5389f786b66d651d31 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T13:27:37,398 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/a/df234d9f50944748ba7b2af6282619b7 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732454852655/Put/seqid=0 2024-11-24T13:27:37,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741858_1035 (size=50463) 2024-11-24T13:27:37,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741858_1035 (size=50463) 2024-11-24T13:27:37,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741858_1035 (size=50463) 2024-11-24T13:27:37,408 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/a/df234d9f50944748ba7b2af6282619b7 2024-11-24T13:27:37,415 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/a/df234d9f50944748ba7b2af6282619b7 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/a/df234d9f50944748ba7b2af6282619b7 2024-11-24T13:27:37,422 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/a/df234d9f50944748ba7b2af6282619b7, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T13:27:37,422 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 338df40f90680e5389f786b66d651d31 in 61ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T13:27:37,438 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T13:27:37,438 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 338df40f90680e5389f786b66d651d31 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T13:27:37,449 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/a/3defad01d3c44fdaac6a4c256fa48f96 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732454852691/Put/seqid=0 2024-11-24T13:27:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741859_1036 (size=20072) 2024-11-24T13:27:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741859_1036 (size=20072) 2024-11-24T13:27:37,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741859_1036 (size=20072) 2024-11-24T13:27:37,458 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/a/3defad01d3c44fdaac6a4c256fa48f96 2024-11-24T13:27:37,487 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/b/f656dc8bec02452d8771015dc243e59f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732454852710/Put/seqid=0 2024-11-24T13:27:37,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741860_1037 (size=35835) 2024-11-24T13:27:37,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741860_1037 (size=35835) 2024-11-24T13:27:37,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741860_1037 (size=35835) 2024-11-24T13:27:37,495 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/b/f656dc8bec02452d8771015dc243e59f 2024-11-24T13:27:37,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/a/3defad01d3c44fdaac6a4c256fa48f96 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/a/3defad01d3c44fdaac6a4c256fa48f96 2024-11-24T13:27:37,508 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/a/3defad01d3c44fdaac6a4c256fa48f96, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T13:27:37,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/b/f656dc8bec02452d8771015dc243e59f as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/b/f656dc8bec02452d8771015dc243e59f 2024-11-24T13:27:37,515 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/b/f656dc8bec02452d8771015dc243e59f, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T13:27:37,516 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 338df40f90680e5389f786b66d651d31 in 78ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T13:27:37,527 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T13:27:37,527 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 338df40f90680e5389f786b66d651d31 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T13:27:37,534 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/b/dd21e0e762e74adab7c0562a22ce3e3c is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732454852726/Put/seqid=0 2024-11-24T13:27:37,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741861_1038 (size=35082) 2024-11-24T13:27:37,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741861_1038 (size=35082) 2024-11-24T13:27:37,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741861_1038 (size=35082) 2024-11-24T13:27:37,545 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/b/dd21e0e762e74adab7c0562a22ce3e3c 2024-11-24T13:27:37,568 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/c/3b38831cf6d847998bc64cfbe823c2cb is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732454852750/Put/seqid=0 2024-11-24T13:27:37,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741862_1039 (size=20825) 2024-11-24T13:27:37,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741862_1039 (size=20825) 2024-11-24T13:27:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741862_1039 (size=20825) 2024-11-24T13:27:37,579 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/c/3b38831cf6d847998bc64cfbe823c2cb 2024-11-24T13:27:37,586 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/b/dd21e0e762e74adab7c0562a22ce3e3c as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/b/dd21e0e762e74adab7c0562a22ce3e3c 2024-11-24T13:27:37,593 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/b/dd21e0e762e74adab7c0562a22ce3e3c, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T13:27:37,594 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/c/3b38831cf6d847998bc64cfbe823c2cb as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/c/3b38831cf6d847998bc64cfbe823c2cb 2024-11-24T13:27:37,600 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/c/3b38831cf6d847998bc64cfbe823c2cb, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T13:27:37,600 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 338df40f90680e5389f786b66d651d31 in 73ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T13:27:37,608 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732454852786/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:37,611 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000003002 2024-11-24T13:27:37,611 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T13:27:37,612 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 338df40f90680e5389f786b66d651d31 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T13:27:37,621 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/c/6c55e14b6e854848aafb68b2ac644f8c is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732454852756/Put/seqid=0 2024-11-24T13:27:37,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741863_1040 (size=50301) 2024-11-24T13:27:37,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741863_1040 (size=50301) 2024-11-24T13:27:37,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741863_1040 (size=50301) 2024-11-24T13:27:37,631 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/c/6c55e14b6e854848aafb68b2ac644f8c 2024-11-24T13:27:37,639 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6c55e14b6e854848aafb68b2ac644f8c 2024-11-24T13:27:37,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/.tmp/c/6c55e14b6e854848aafb68b2ac644f8c as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/c/6c55e14b6e854848aafb68b2ac644f8c 2024-11-24T13:27:37,646 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6c55e14b6e854848aafb68b2ac644f8c 2024-11-24T13:27:37,646 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/c/6c55e14b6e854848aafb68b2ac644f8c, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T13:27:37,647 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 338df40f90680e5389f786b66d651d31 in 34ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T13:27:37,647 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/0000000000000003002 2024-11-24T13:27:37,648 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:37,648 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:37,649 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T13:27:37,651 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 338df40f90680e5389f786b66d651d31 2024-11-24T13:27:37,654 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenIntoWAL/338df40f90680e5389f786b66d651d31/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T13:27:37,655 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 338df40f90680e5389f786b66d651d31; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66993190, jitterRate=-0.0017236769199371338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T13:27:37,655 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 338df40f90680e5389f786b66d651d31: Writing region info on filesystem at 1732454856949Initializing all the Stores at 1732454856950 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454856950Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454856951 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454856951Cleaning up temporary data from old regions at 1732454857648 (+697 ms)Region opened successfully at 1732454857655 (+7 ms) 2024-11-24T13:27:37,724 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 338df40f90680e5389f786b66d651d31, disabling compactions & flushes 2024-11-24T13:27:37,724 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:37,724 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:37,724 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. after waiting 0 ms 2024-11-24T13:27:37,724 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:37,726 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732454852564.338df40f90680e5389f786b66d651d31. 2024-11-24T13:27:37,726 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 338df40f90680e5389f786b66d651d31: Waiting for close lock at 1732454857724Disabling compacts and flushes for region at 1732454857724Disabling writes for close at 1732454857724Writing region close event to WAL at 1732454857726 (+2 ms)Closed at 1732454857726 2024-11-24T13:27:37,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741857_1034 (size=93) 2024-11-24T13:27:37,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741857_1034 (size=93) 2024-11-24T13:27:37,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741857_1034 (size=93) 2024-11-24T13:27:37,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:37,734 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732454856925) 2024-11-24T13:27:37,757 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=402 (was 388) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35337 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40461 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40595 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:35337 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-176622288_22 at /127.0.0.1:53358 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-176622288_22 at /127.0.0.1:47448 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-176622288_22 at /127.0.0.1:51048 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:40595 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:40461 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-176622288_22 at /127.0.0.1:47468 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 167) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7427 (was 7483) 2024-11-24T13:27:37,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=7426 2024-11-24T13:27:37,788 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:37,790 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:37,790 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:37,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37301864, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-37301864, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:37,815 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37301864/hregion-37301864.1732454857794, exclude list is [], retry=0 2024-11-24T13:27:37,818 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:37,818 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:37,819 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:37,821 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37301864/hregion-37301864.1732454857794 2024-11-24T13:27:37,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:37,822 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dd0f51f5d11d6295aed46dd54bb99765, NAME => 'test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:37,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741865_1042 (size=43) 2024-11-24T13:27:37,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741865_1042 (size=43) 2024-11-24T13:27:37,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741865_1042 (size=43) 2024-11-24T13:27:37,834 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:37,835 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,837 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0f51f5d11d6295aed46dd54bb99765 columnFamilyName a 2024-11-24T13:27:37,837 DEBUG [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:37,837 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(327): Store=dd0f51f5d11d6295aed46dd54bb99765/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:37,838 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,839 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0f51f5d11d6295aed46dd54bb99765 columnFamilyName b 2024-11-24T13:27:37,839 DEBUG [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:37,840 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(327): Store=dd0f51f5d11d6295aed46dd54bb99765/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:37,840 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,842 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0f51f5d11d6295aed46dd54bb99765 columnFamilyName c 2024-11-24T13:27:37,842 DEBUG [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:37,842 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(327): Store=dd0f51f5d11d6295aed46dd54bb99765/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:37,843 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,844 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,846 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,846 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:37,848 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:37,853 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:37,853 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dd0f51f5d11d6295aed46dd54bb99765; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75016346, jitterRate=0.11783066391944885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:37,854 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dd0f51f5d11d6295aed46dd54bb99765: Writing region info on filesystem at 1732454857834Initializing all the Stores at 1732454857835 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454857835Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454857835Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454857835Cleaning up temporary data from old regions at 1732454857846 (+11 ms)Region opened successfully at 1732454857854 (+8 ms) 2024-11-24T13:27:37,854 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dd0f51f5d11d6295aed46dd54bb99765, disabling compactions & flushes 2024-11-24T13:27:37,854 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:37,854 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:37,855 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. after waiting 0 ms 2024-11-24T13:27:37,855 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:37,855 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:37,855 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dd0f51f5d11d6295aed46dd54bb99765: Waiting for close lock at 1732454857854Disabling compacts and flushes for region at 1732454857854Disabling writes for close at 1732454857855 (+1 ms)Writing region close event to WAL at 1732454857855Closed at 1732454857855 2024-11-24T13:27:37,859 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-37301864/hregion-37301864.1732454857794 not finished, retry = 0 2024-11-24T13:27:37,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741864_1041 (size=93) 2024-11-24T13:27:37,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741864_1041 (size=93) 2024-11-24T13:27:37,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741864_1041 (size=93) 2024-11-24T13:27:37,962 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:37,962 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37301864:(num 1732454857794) 2024-11-24T13:27:37,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:37,964 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:37,978 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965, exclude list is [], retry=0 2024-11-24T13:27:37,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:37,982 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:37,982 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:37,983 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 2024-11-24T13:27:37,984 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:38,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741866_1043 (size=344306) 2024-11-24T13:27:38,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741866_1043 (size=344306) 2024-11-24T13:27:38,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741866_1043 (size=344306) 2024-11-24T13:27:38,120 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965, size=336.2 K (344306bytes) 2024-11-24T13:27:38,120 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 2024-11-24T13:27:38,120 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 after 0ms 2024-11-24T13:27:38,124 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 took 4ms 2024-11-24T13:27:38,128 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732454857965.temp 2024-11-24T13:27:38,129 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000000001-wal.1732454857965.temp 2024-11-24T13:27:38,148 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 so closing down 2024-11-24T13:27:38,148 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:38,148 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:38,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741867_1044 (size=344306) 2024-11-24T13:27:38,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741867_1044 (size=344306) 2024-11-24T13:27:38,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741867_1044 (size=344306) 2024-11-24T13:27:38,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000000001-wal.1732454857965.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-24T13:27:38,160 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000000001-wal.1732454857965.temp to hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003000 2024-11-24T13:27:38,160 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-24T13:27:38,161 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965, journal: Splitting hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965, size=336.2 K (344306bytes) at 1732454858120Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000000001-wal.1732454857965.temp at 1732454858129 (+9 ms)Split 1024 edits, skipped 0 edits. at 1732454858133 (+4 ms)Split 2048 edits, skipped 0 edits. at 1732454858141 (+8 ms)Finishing writing output for hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 so closing down at 1732454858148 (+7 ms)3 split writer threads finished at 1732454858148Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000000001-wal.1732454857965.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1732454858158 (+10 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000000001-wal.1732454857965.temp to hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003000 at 1732454858160 (+2 ms)Processed 3000 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732454858160 2024-11-24T13:27:38,163 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454857965 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454857965 2024-11-24T13:27:38,164 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003000 2024-11-24T13:27:38,164 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:38,166 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:38,188 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167, exclude list is [], retry=0 2024-11-24T13:27:38,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:38,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:38,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:38,194 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 2024-11-24T13:27:38,194 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741868_1045 (size=344433) 2024-11-24T13:27:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741868_1045 (size=344433) 2024-11-24T13:27:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741868_1045 (size=344433) 2024-11-24T13:27:38,346 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167, size=336.4 K (344433bytes) 2024-11-24T13:27:38,346 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 2024-11-24T13:27:38,346 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 after 0ms 2024-11-24T13:27:38,351 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 took 6ms 2024-11-24T13:27:38,355 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732454858167.temp 2024-11-24T13:27:38,357 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003001-wal.1732454858167.temp 2024-11-24T13:27:38,384 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 so closing down 2024-11-24T13:27:38,384 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:38,384 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:38,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741869_1046 (size=344433) 2024-11-24T13:27:38,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741869_1046 (size=344433) 2024-11-24T13:27:38,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741869_1046 (size=344433) 2024-11-24T13:27:38,389 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003001-wal.1732454858167.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-11-24T13:27:38,391 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003001-wal.1732454858167.temp to hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000006000 2024-11-24T13:27:38,391 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-24T13:27:38,391 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167, journal: Splitting hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167, size=336.4 K (344433bytes) at 1732454858346Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003001-wal.1732454858167.temp at 1732454858357 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732454858363 (+6 ms)Split 2048 edits, skipped 0 edits. at 1732454858374 (+11 ms)Finishing writing output for hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 so closing down at 1732454858384 (+10 ms)3 split writer threads finished at 1732454858384Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003001-wal.1732454858167.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1732454858389 (+5 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003001-wal.1732454858167.temp to hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000006000 at 1732454858391 (+2 ms)Processed 3000 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732454858391 2024-11-24T13:27:38,393 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858167 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454858167 2024-11-24T13:27:38,395 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000006000 2024-11-24T13:27:38,395 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:38,397 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/test2727-manual,16010,1732454857787, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:38,412 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858397, exclude list is [], retry=0 2024-11-24T13:27:38,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:38,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:38,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:38,418 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732454857787/wal.1732454858397 2024-11-24T13:27:38,419 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:38,419 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dd0f51f5d11d6295aed46dd54bb99765, NAME => 'test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:38,419 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:38,419 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,419 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,421 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,422 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0f51f5d11d6295aed46dd54bb99765 columnFamilyName a 2024-11-24T13:27:38,422 DEBUG [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:38,423 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(327): Store=dd0f51f5d11d6295aed46dd54bb99765/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:38,423 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,424 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0f51f5d11d6295aed46dd54bb99765 columnFamilyName b 2024-11-24T13:27:38,424 DEBUG [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:38,425 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(327): Store=dd0f51f5d11d6295aed46dd54bb99765/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:38,425 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,426 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0f51f5d11d6295aed46dd54bb99765 columnFamilyName c 2024-11-24T13:27:38,426 DEBUG [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:38,427 INFO [StoreOpener-dd0f51f5d11d6295aed46dd54bb99765-1 {}] regionserver.HStore(327): Store=dd0f51f5d11d6295aed46dd54bb99765/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:38,427 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,428 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,430 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,431 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003000 2024-11-24T13:27:38,465 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003000 2024-11-24T13:27:38,467 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000006000 2024-11-24T13:27:38,512 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000006000 2024-11-24T13:27:38,512 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dd0f51f5d11d6295aed46dd54bb99765 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T13:27:38,535 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/a/c7bb94ce08994ebda5cf32b541e0d4c7 is 41, key is test2727/a:100/1732454858198/Put/seqid=0 2024-11-24T13:27:38,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741871_1048 (size=84227) 2024-11-24T13:27:38,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741871_1048 (size=84227) 2024-11-24T13:27:38,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741871_1048 (size=84227) 2024-11-24T13:27:38,544 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/a/c7bb94ce08994ebda5cf32b541e0d4c7 2024-11-24T13:27:38,583 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/b/b893a3f8739d47c7b3567240524aaec7 is 41, key is test2727/b:100/1732454858241/Put/seqid=0 2024-11-24T13:27:38,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741872_1049 (size=84609) 2024-11-24T13:27:38,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741872_1049 (size=84609) 2024-11-24T13:27:38,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741872_1049 (size=84609) 2024-11-24T13:27:38,591 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/b/b893a3f8739d47c7b3567240524aaec7 2024-11-24T13:27:38,619 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/c/b83b2026db1c439cbae0bc0c9699b375 is 41, key is test2727/c:100/1732454858285/Put/seqid=0 2024-11-24T13:27:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741873_1050 (size=84609) 2024-11-24T13:27:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741873_1050 (size=84609) 2024-11-24T13:27:38,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741873_1050 (size=84609) 2024-11-24T13:27:38,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/c/b83b2026db1c439cbae0bc0c9699b375 2024-11-24T13:27:38,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/a/c7bb94ce08994ebda5cf32b541e0d4c7 as hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/a/c7bb94ce08994ebda5cf32b541e0d4c7 2024-11-24T13:27:38,641 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/a/c7bb94ce08994ebda5cf32b541e0d4c7, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T13:27:38,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/b/b893a3f8739d47c7b3567240524aaec7 as hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/b/b893a3f8739d47c7b3567240524aaec7 2024-11-24T13:27:38,648 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/b/b893a3f8739d47c7b3567240524aaec7, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T13:27:38,649 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/.tmp/c/b83b2026db1c439cbae0bc0c9699b375 as hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/c/b83b2026db1c439cbae0bc0c9699b375 2024-11-24T13:27:38,656 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/c/b83b2026db1c439cbae0bc0c9699b375, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T13:27:38,657 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for dd0f51f5d11d6295aed46dd54bb99765 in 144ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T13:27:38,657 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000003000 2024-11-24T13:27:38,658 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/0000000000000006000 2024-11-24T13:27:38,659 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,659 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,660 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:38,662 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dd0f51f5d11d6295aed46dd54bb99765 2024-11-24T13:27:38,664 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/test2727/dd0f51f5d11d6295aed46dd54bb99765/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T13:27:38,665 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dd0f51f5d11d6295aed46dd54bb99765; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74399722, jitterRate=0.10864225029945374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:38,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dd0f51f5d11d6295aed46dd54bb99765: Writing region info on filesystem at 1732454858419Initializing all the Stores at 1732454858420 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454858420Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454858420Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454858420Obtaining lock to block concurrent updates at 1732454858512 (+92 ms)Preparing flush snapshotting stores in dd0f51f5d11d6295aed46dd54bb99765 at 1732454858512Finished memstore snapshotting test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732454858513 (+1 ms)Flushing stores of test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. at 1732454858513Flushing dd0f51f5d11d6295aed46dd54bb99765/a: creating writer at 1732454858513Flushing dd0f51f5d11d6295aed46dd54bb99765/a: appending metadata at 1732454858535 (+22 ms)Flushing dd0f51f5d11d6295aed46dd54bb99765/a: closing flushed file at 1732454858535Flushing dd0f51f5d11d6295aed46dd54bb99765/b: creating writer at 1732454858552 (+17 ms)Flushing dd0f51f5d11d6295aed46dd54bb99765/b: appending metadata at 1732454858581 (+29 ms)Flushing dd0f51f5d11d6295aed46dd54bb99765/b: closing flushed file at 1732454858581Flushing dd0f51f5d11d6295aed46dd54bb99765/c: creating writer at 1732454858598 (+17 ms)Flushing dd0f51f5d11d6295aed46dd54bb99765/c: appending metadata at 1732454858619 (+21 ms)Flushing dd0f51f5d11d6295aed46dd54bb99765/c: closing flushed file at 1732454858619Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d93de0b: reopening flushed file at 1732454858633 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2702ee51: reopening flushed file at 1732454858641 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@568af8da: reopening flushed file at 1732454858648 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for dd0f51f5d11d6295aed46dd54bb99765 in 144ms, sequenceid=6000, compaction requested=false; wal=null at 1732454858657 (+9 ms)Cleaning up temporary data from old regions at 1732454858659 (+2 ms)Region opened successfully at 1732454858666 (+7 ms) 2024-11-24T13:27:38,667 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T13:27:38,667 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dd0f51f5d11d6295aed46dd54bb99765, disabling compactions & flushes 2024-11-24T13:27:38,667 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:38,667 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:38,667 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. after waiting 0 ms 2024-11-24T13:27:38,668 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:38,669 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732454857789.dd0f51f5d11d6295aed46dd54bb99765. 2024-11-24T13:27:38,669 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dd0f51f5d11d6295aed46dd54bb99765: Waiting for close lock at 1732454858667Disabling compacts and flushes for region at 1732454858667Disabling writes for close at 1732454858667Writing region close event to WAL at 1732454858669 (+2 ms)Closed at 1732454858669 2024-11-24T13:27:38,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741870_1047 (size=93) 2024-11-24T13:27:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741870_1047 (size=93) 2024-11-24T13:27:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741870_1047 (size=93) 2024-11-24T13:27:38,676 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:38,676 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732454858397) 2024-11-24T13:27:38,689 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:47570 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53358 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:51158 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=7292 (was 7426) 2024-11-24T13:27:38,703 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=7291 2024-11-24T13:27:38,725 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:38,731 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:38,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732454858732 2024-11-24T13:27:38,739 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732 2024-11-24T13:27:38,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:38,742 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cf47ec799bc905b018801cfa66544fa1, NAME => 'testSequentialEditLogSeqNum,,1732454858725.cf47ec799bc905b018801cfa66544fa1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:38,742 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732454858725.cf47ec799bc905b018801cfa66544fa1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:38,742 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,742 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,743 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1 doesn't exist for region: cf47ec799bc905b018801cfa66544fa1 on table testSequentialEditLogSeqNum 2024-11-24T13:27:38,743 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: cf47ec799bc905b018801cfa66544fa1 on table testSequentialEditLogSeqNum 2024-11-24T13:27:38,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741875_1052 (size=62) 2024-11-24T13:27:38,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741875_1052 (size=62) 2024-11-24T13:27:38,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741875_1052 (size=62) 2024-11-24T13:27:38,755 INFO [StoreOpener-cf47ec799bc905b018801cfa66544fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,756 INFO [StoreOpener-cf47ec799bc905b018801cfa66544fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf47ec799bc905b018801cfa66544fa1 columnFamilyName a 2024-11-24T13:27:38,756 DEBUG [StoreOpener-cf47ec799bc905b018801cfa66544fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:38,757 INFO [StoreOpener-cf47ec799bc905b018801cfa66544fa1-1 {}] regionserver.HStore(327): Store=cf47ec799bc905b018801cfa66544fa1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:38,757 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,758 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T13:27:38,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:38,759 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,759 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,761 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:27:38,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T13:27:38,761 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cf47ec799bc905b018801cfa66544fa1 2024-11-24T13:27:38,761 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T13:27:38,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:38,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T13:27:38,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:38,763 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:38,763 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cf47ec799bc905b018801cfa66544fa1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58812906, jitterRate=-0.12361940741539001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:38,765 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cf47ec799bc905b018801cfa66544fa1: Writing region info on filesystem at 1732454858742Initializing all the Stores at 1732454858754 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454858754Cleaning up temporary data from old regions at 1732454858759 (+5 ms)Region opened successfully at 1732454858764 (+5 ms) 2024-11-24T13:27:38,776 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cf47ec799bc905b018801cfa66544fa1 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T13:27:38,799 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/.tmp/a/58cd634f569d40329e02ec116d87e2eb is 81, key is testSequentialEditLogSeqNum/a:x0/1732454858765/Put/seqid=0 2024-11-24T13:27:38,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741876_1053 (size=5833) 2024-11-24T13:27:38,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741876_1053 (size=5833) 2024-11-24T13:27:38,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741876_1053 (size=5833) 2024-11-24T13:27:38,807 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/.tmp/a/58cd634f569d40329e02ec116d87e2eb 2024-11-24T13:27:38,814 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/.tmp/a/58cd634f569d40329e02ec116d87e2eb as hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/a/58cd634f569d40329e02ec116d87e2eb 2024-11-24T13:27:38,821 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/a/58cd634f569d40329e02ec116d87e2eb, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T13:27:38,822 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for cf47ec799bc905b018801cfa66544fa1 in 46ms, sequenceid=13, compaction requested=false 2024-11-24T13:27:38,823 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cf47ec799bc905b018801cfa66544fa1: 2024-11-24T13:27:38,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:27:38,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:27:38,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:27:38,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:27:38,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:27:38,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741874_1051 (size=3017) 2024-11-24T13:27:38,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741874_1051 (size=3017) 2024-11-24T13:27:38,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741874_1051 (size=3017) 2024-11-24T13:27:38,860 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732, size=2.9 K (3017bytes) 2024-11-24T13:27:38,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732 2024-11-24T13:27:38,861 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732 after 1ms 2024-11-24T13:27:38,864 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732 took 4ms 2024-11-24T13:27:38,867 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732 so closing down 2024-11-24T13:27:38,867 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:38,868 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732454858732.temp 2024-11-24T13:27:38,870 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000003-wal.1732454858732.temp 2024-11-24T13:27:38,870 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741877_1054 (size=2433) 2024-11-24T13:27:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741877_1054 (size=2433) 2024-11-24T13:27:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741877_1054 (size=2433) 2024-11-24T13:27:38,879 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000003-wal.1732454858732.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:38,881 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000003-wal.1732454858732.temp to hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000020 2024-11-24T13:27:38,881 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-24T13:27:38,882 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732, journal: Splitting hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732, size=2.9 K (3017bytes) at 1732454858860Finishing writing output for hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732 so closing down at 1732454858867 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000003-wal.1732454858732.temp at 1732454858870 (+3 ms)3 split writer threads finished at 1732454858870Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000003-wal.1732454858732.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732454858879 (+9 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000003-wal.1732454858732.temp to hdfs://localhost:35337/hbase/data/default/testSequentialEditLogSeqNum/cf47ec799bc905b018801cfa66544fa1/recovered.edits/0000000000000000020 at 1732454858881 (+2 ms)Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:35337/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732454858724/wal.1732454858732, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732454858881 2024-11-24T13:27:38,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53358 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:51158 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=7274 (was 7291) 2024-11-24T13:27:38,911 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=7272 2024-11-24T13:27:38,929 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:38,932 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:38,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:38,969 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-60052579, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-60052579, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:38,991 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-60052579/hregion-60052579.1732454858969, exclude list is [], retry=0 2024-11-24T13:27:38,995 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:38,996 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:38,996 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:38,999 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-60052579/hregion-60052579.1732454858969 2024-11-24T13:27:38,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:38,999 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 97f82956fb68e9c05d3fb8403f1559f6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741879_1056 (size=70) 2024-11-24T13:27:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741879_1056 (size=70) 2024-11-24T13:27:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741879_1056 (size=70) 2024-11-24T13:27:39,016 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:39,018 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,020 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName a 2024-11-24T13:27:39,020 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:39,021 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:39,021 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,023 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName b 2024-11-24T13:27:39,023 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:39,023 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:39,024 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,026 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName c 2024-11-24T13:27:39,026 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:39,026 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:39,026 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,028 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,029 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,029 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,030 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:39,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,035 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:39,035 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97f82956fb68e9c05d3fb8403f1559f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59156564, jitterRate=-0.11849850416183472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:39,037 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97f82956fb68e9c05d3fb8403f1559f6: Writing region info on filesystem at 1732454859016Initializing all the Stores at 1732454859017 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454859017Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454859018 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454859018Cleaning up temporary data from old regions at 1732454859029 (+11 ms)Region opened successfully at 1732454859037 (+8 ms) 2024-11-24T13:27:39,037 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97f82956fb68e9c05d3fb8403f1559f6, disabling compactions & flushes 2024-11-24T13:27:39,037 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:39,037 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:39,037 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. after waiting 0 ms 2024-11-24T13:27:39,037 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:39,038 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:39,038 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97f82956fb68e9c05d3fb8403f1559f6: Waiting for close lock at 1732454859037Disabling compacts and flushes for region at 1732454859037Disabling writes for close at 1732454859037Writing region close event to WAL at 1732454859038 (+1 ms)Closed at 1732454859038 2024-11-24T13:27:39,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741878_1055 (size=93) 2024-11-24T13:27:39,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741878_1055 (size=93) 2024-11-24T13:27:39,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741878_1055 (size=93) 2024-11-24T13:27:39,046 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:39,046 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-60052579:(num 1732454858969) 2024-11-24T13:27:39,046 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:39,049 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:39,070 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050, exclude list is [], retry=0 2024-11-24T13:27:39,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:39,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:39,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:39,077 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 2024-11-24T13:27:39,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:39,079 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97f82956fb68e9c05d3fb8403f1559f6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:39,080 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:39,080 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,080 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,082 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,083 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName a 2024-11-24T13:27:39,083 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:39,084 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:39,084 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,085 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName b 2024-11-24T13:27:39,085 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:39,086 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:39,086 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,087 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName c 2024-11-24T13:27:39,087 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:39,088 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:39,088 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,088 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,090 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,091 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,091 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,092 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:39,093 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:39,094 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97f82956fb68e9c05d3fb8403f1559f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67853849, jitterRate=0.011101141571998596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:39,096 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97f82956fb68e9c05d3fb8403f1559f6: Writing region info on filesystem at 1732454859080Initializing all the Stores at 1732454859081 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454859081Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454859081Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454859081Cleaning up temporary data from old regions at 1732454859091 (+10 ms)Region opened successfully at 1732454859095 (+4 ms) 2024-11-24T13:27:39,100 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732454859099/Put/seqid=0 2024-11-24T13:27:39,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741881_1058 (size=4826) 2024-11-24T13:27:39,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741881_1058 (size=4826) 2024-11-24T13:27:39,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741881_1058 (size=4826) 2024-11-24T13:27:39,109 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35337/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 97f82956fb68e9c05d3fb8403f1559f6/a 2024-11-24T13:27:39,117 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T13:27:39,118 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T13:27:39,118 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 97f82956fb68e9c05d3fb8403f1559f6: 2024-11-24T13:27:39,120 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/08d8eb54ef3644a1811e245859450c72_SeqId_3_ 2024-11-24T13:27:39,121 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35337/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 97f82956fb68e9c05d3fb8403f1559f6/a as hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/08d8eb54ef3644a1811e245859450c72_SeqId_3_ - updating store file list. 2024-11-24T13:27:39,127 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 08d8eb54ef3644a1811e245859450c72_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T13:27:39,128 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/08d8eb54ef3644a1811e245859450c72_SeqId_3_ into 97f82956fb68e9c05d3fb8403f1559f6/a 2024-11-24T13:27:39,128 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35337/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 97f82956fb68e9c05d3fb8403f1559f6/a (new location: hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/08d8eb54ef3644a1811e245859450c72_SeqId_3_) 2024-11-24T13:27:39,172 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050, size=0 (0bytes) 2024-11-24T13:27:39,172 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 might be still open, length is 0 2024-11-24T13:27:39,172 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 2024-11-24T13:27:39,173 WARN [IPC Server handler 1 on default port 35337 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T13:27:39,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 after 1ms 2024-11-24T13:27:39,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:51246 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36111:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51246 dst: /127.0.0.1:36111 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36111 remote=/127.0.0.1:51246]. Total timeout mills is 60000, 59858 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:39,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:53526 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53526 dst: /127.0.0.1:46321 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:39,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:47678 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47678 dst: /127.0.0.1:45659 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:39,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741880_1059 (size=504) 2024-11-24T13:27:39,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741880_1059 (size=504) 2024-11-24T13:27:43,174 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 after 4002ms 2024-11-24T13:27:43,178 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 took 4006ms 2024-11-24T13:27:43,180 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050; continuing. 2024-11-24T13:27:43,180 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 so closing down 2024-11-24T13:27:43,180 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:43,182 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732454859050.temp 2024-11-24T13:27:43,183 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005-wal.1732454859050.temp 2024-11-24T13:27:43,183 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741882_1060 (size=238) 2024-11-24T13:27:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741882_1060 (size=238) 2024-11-24T13:27:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741882_1060 (size=238) 2024-11-24T13:27:43,192 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005-wal.1732454859050.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:43,193 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005-wal.1732454859050.temp to hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005 2024-11-24T13:27:43,193 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T13:27:43,194 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050, journal: Splitting hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050, size=0 (0bytes) at 1732454859172Finishing writing output for hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 so closing down at 1732454863180 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005-wal.1732454859050.temp at 1732454863183 (+3 ms)3 split writer threads finished at 1732454863183Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005-wal.1732454859050.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732454863192 (+9 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005-wal.1732454859050.temp to hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005 at 1732454863193 (+1 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050, size=0, length=0, corrupted=false, cancelled=false at 1732454863194 (+1 ms) 2024-11-24T13:27:43,195 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454859050 2024-11-24T13:27:43,196 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005 2024-11-24T13:27:43,196 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:43,198 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:43,213 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454863199, exclude list is [], retry=0 2024-11-24T13:27:43,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:43,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:43,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:43,218 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454863199 2024-11-24T13:27:43,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:43,219 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97f82956fb68e9c05d3fb8403f1559f6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:43,219 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:43,219 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,219 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,221 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,222 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName a 2024-11-24T13:27:43,222 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:43,230 DEBUG [StoreFileOpener-97f82956fb68e9c05d3fb8403f1559f6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 08d8eb54ef3644a1811e245859450c72_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T13:27:43,230 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/08d8eb54ef3644a1811e245859450c72_SeqId_3_ 2024-11-24T13:27:43,230 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:43,230 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,231 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName b 2024-11-24T13:27:43,231 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:43,232 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:43,232 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,233 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f82956fb68e9c05d3fb8403f1559f6 columnFamilyName c 2024-11-24T13:27:43,233 DEBUG [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:43,233 INFO [StoreOpener-97f82956fb68e9c05d3fb8403f1559f6-1 {}] regionserver.HStore(327): Store=97f82956fb68e9c05d3fb8403f1559f6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:43,234 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,234 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,237 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005 2024-11-24T13:27:43,240 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005 2024-11-24T13:27:43,240 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97f82956fb68e9c05d3fb8403f1559f6 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T13:27:43,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/.tmp/a/3552850a257a4f53bc3ca3fea0c7d873 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732454859134/Put/seqid=0 2024-11-24T13:27:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741884_1062 (size=5149) 2024-11-24T13:27:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741884_1062 (size=5149) 2024-11-24T13:27:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741884_1062 (size=5149) 2024-11-24T13:27:43,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/.tmp/a/3552850a257a4f53bc3ca3fea0c7d873 2024-11-24T13:27:43,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/.tmp/a/3552850a257a4f53bc3ca3fea0c7d873 as hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/3552850a257a4f53bc3ca3fea0c7d873 2024-11-24T13:27:43,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/a/3552850a257a4f53bc3ca3fea0c7d873, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T13:27:43,277 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 97f82956fb68e9c05d3fb8403f1559f6 in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T13:27:43,278 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/0000000000000000005 2024-11-24T13:27:43,279 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,279 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,280 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:43,282 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97f82956fb68e9c05d3fb8403f1559f6 2024-11-24T13:27:43,284 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/97f82956fb68e9c05d3fb8403f1559f6/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T13:27:43,285 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97f82956fb68e9c05d3fb8403f1559f6; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67846587, jitterRate=0.010992929339408875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:43,285 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97f82956fb68e9c05d3fb8403f1559f6: Writing region info on filesystem at 1732454863220Initializing all the Stores at 1732454863221 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454863221Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454863221Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454863221Obtaining lock to block concurrent updates at 1732454863240 (+19 ms)Preparing flush snapshotting stores in 97f82956fb68e9c05d3fb8403f1559f6 at 1732454863240Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732454863241 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. at 1732454863241Flushing 97f82956fb68e9c05d3fb8403f1559f6/a: creating writer at 1732454863241Flushing 97f82956fb68e9c05d3fb8403f1559f6/a: appending metadata at 1732454863256 (+15 ms)Flushing 97f82956fb68e9c05d3fb8403f1559f6/a: closing flushed file at 1732454863256Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cc119d3: reopening flushed file at 1732454863270 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 97f82956fb68e9c05d3fb8403f1559f6 in 37ms, sequenceid=5, compaction requested=false; wal=null at 1732454863277 (+7 ms)Cleaning up temporary data from old regions at 1732454863279 (+2 ms)Region opened successfully at 1732454863285 (+6 ms) 2024-11-24T13:27:43,290 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97f82956fb68e9c05d3fb8403f1559f6, disabling compactions & flushes 2024-11-24T13:27:43,290 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:43,290 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:43,290 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. after waiting 0 ms 2024-11-24T13:27:43,290 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:43,291 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732454858930.97f82956fb68e9c05d3fb8403f1559f6. 2024-11-24T13:27:43,291 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97f82956fb68e9c05d3fb8403f1559f6: Waiting for close lock at 1732454863290Disabling compacts and flushes for region at 1732454863290Disabling writes for close at 1732454863290Writing region close event to WAL at 1732454863291 (+1 ms)Closed at 1732454863291 2024-11-24T13:27:43,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741883_1061 (size=93) 2024-11-24T13:27:43,294 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454863199 not finished, retry = 0 2024-11-24T13:27:43,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741883_1061 (size=93) 2024-11-24T13:27:43,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741883_1061 (size=93) 2024-11-24T13:27:43,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:43,397 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732454863199) 2024-11-24T13:27:43,413 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171536292_22 at /127.0.0.1:43828 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171536292_22 at /127.0.0.1:52108 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171536292_22 at /127.0.0.1:42634 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35337 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:35337 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=7225 (was 7272) 2024-11-24T13:27:43,425 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=7223 2024-11-24T13:27:43,442 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:43,448 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T13:27:43,452 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b10924491aa5,38765,1732454848333 2024-11-24T13:27:43,455 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45d9b6e5 2024-11-24T13:27:43,456 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T13:27:43,458 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T13:27:43,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T13:27:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T13:27:43,470 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T13:27:43,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T13:27:43,473 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:43,474 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T13:27:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:27:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741885_1063 (size=694) 2024-11-24T13:27:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741885_1063 (size=694) 2024-11-24T13:27:43,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741885_1063 (size=694) 2024-11-24T13:27:43,492 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07 2024-11-24T13:27:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741886_1064 (size=77) 2024-11-24T13:27:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741886_1064 (size=77) 2024-11-24T13:27:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741886_1064 (size=77) 2024-11-24T13:27:43,504 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:43,504 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:27:43,504 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,504 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,504 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:27:43,504 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,504 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,504 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454863504Disabling compacts and flushes for region at 1732454863504Disabling writes for close at 1732454863504Writing region close event to WAL at 1732454863504Closed at 1732454863504 2024-11-24T13:27:43,506 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T13:27:43,513 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732454863506"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732454863506"}]},"ts":"1732454863506"} 2024-11-24T13:27:43,516 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T13:27:43,518 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T13:27:43,521 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732454863518"}]},"ts":"1732454863518"} 2024-11-24T13:27:43,525 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T13:27:43,525 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {b10924491aa5=0} racks are {/default-rack=0} 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T13:27:43,527 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:27:43,527 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:27:43,527 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T13:27:43,527 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:27:43,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN}] 2024-11-24T13:27:43,530 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN 2024-11-24T13:27:43,531 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN; state=OFFLINE, location=b10924491aa5,34273,1732454849133; forceNewPlan=false, retain=false 2024-11-24T13:27:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:27:43,684 INFO [b10924491aa5:38765 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T13:27:43,685 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPENING, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:43,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN because future has completed 2024-11-24T13:27:43,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133}] 2024-11-24T13:27:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:27:43,847 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,847 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:43,847 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,847 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:43,848 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,848 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,849 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,851 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf1 2024-11-24T13:27:43,851 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:43,851 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:43,852 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,853 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf2 2024-11-24T13:27:43,853 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:43,854 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:43,854 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,855 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,855 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,856 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,856 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,857 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T13:27:43,859 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,861 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:43,862 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened abd2f3ec3c78b40d57714a27dadd50db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60884212, jitterRate=-0.0927545428276062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T13:27:43,862 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:43,863 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for abd2f3ec3c78b40d57714a27dadd50db: Running coprocessor pre-open hook at 1732454863848Writing region info on filesystem at 1732454863848Initializing all the Stores at 1732454863849 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454863849Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454863849Cleaning up temporary data from old regions at 1732454863856 (+7 ms)Running coprocessor post-open hooks at 1732454863862 (+6 ms)Region opened successfully at 1732454863863 (+1 ms) 2024-11-24T13:27:43,866 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., pid=6, masterSystemTime=1732454863841 2024-11-24T13:27:43,869 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,869 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:43,870 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPEN, openSeqNum=2, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:43,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133 because future has completed 2024-11-24T13:27:43,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T13:27:43,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133 in 186 msec 2024-11-24T13:27:43,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T13:27:43,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN in 352 msec 2024-11-24T13:27:43,884 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T13:27:43,885 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732454863884"}]},"ts":"1732454863884"} 2024-11-24T13:27:43,888 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T13:27:43,889 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T13:27:43,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 426 msec 2024-11-24T13:27:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:27:44,109 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T13:27:44,109 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T13:27:44,110 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T13:27:44,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T13:27:44,116 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T13:27:44,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T13:27:44,130 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=2] 2024-11-24T13:27:44,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=abd2f3ec3c78b40d57714a27dadd50db, source=b10924491aa5,34273,1732454849133, destination=b10924491aa5,38813,1732454849031, warming up region on b10924491aa5,38813,1732454849031 2024-11-24T13:27:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:27:44,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=abd2f3ec3c78b40d57714a27dadd50db, source=b10924491aa5,34273,1732454849133, destination=b10924491aa5,38813,1732454849031, running balancer 2024-11-24T13:27:44,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42627, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:27:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE 2024-11-24T13:27:44,150 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE 2024-11-24T13:27:44,153 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=CLOSING, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:44,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(7855): Warmup {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:44,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE because future has completed 2024-11-24T13:27:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:44,156 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T13:27:44,156 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133}] 2024-11-24T13:27:44,157 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf1 2024-11-24T13:27:44,158 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:44,158 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:44,158 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,159 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf2 2024-11-24T13:27:44,159 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:44,160 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:27:44,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:27:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454864160Disabling compacts and flushes for region at 1732454864160Disabling writes for close at 1732454864160Writing region close event to WAL at 1732454864161 (+1 ms)Closed at 1732454864161 2024-11-24T13:27:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T13:27:44,316 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,316 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T13:27:44,317 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:27:44,317 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,317 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,317 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:27:44,317 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,317 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing abd2f3ec3c78b40d57714a27dadd50db 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T13:27:44,336 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/53b2858e0df6445fbbbc4f06125cea77 is 35, key is r1/cf1:q/1732454864132/Put/seqid=0 2024-11-24T13:27:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741887_1065 (size=4783) 2024-11-24T13:27:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741887_1065 (size=4783) 2024-11-24T13:27:44,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741887_1065 (size=4783) 2024-11-24T13:27:44,347 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/53b2858e0df6445fbbbc4f06125cea77 2024-11-24T13:27:44,355 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/53b2858e0df6445fbbbc4f06125cea77 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77 2024-11-24T13:27:44,361 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T13:27:44,362 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for abd2f3ec3c78b40d57714a27dadd50db in 45ms, sequenceid=5, compaction requested=false 2024-11-24T13:27:44,362 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T13:27:44,367 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T13:27:44,370 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,370 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454864317Running coprocessor pre-close hooks at 1732454864317Disabling compacts and flushes for region at 1732454864317Disabling writes for close at 1732454864317Obtaining lock to block concurrent updates at 1732454864317Preparing flush snapshotting stores in abd2f3ec3c78b40d57714a27dadd50db at 1732454864317Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732454864318 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. at 1732454864319 (+1 ms)Flushing abd2f3ec3c78b40d57714a27dadd50db/cf1: creating writer at 1732454864319Flushing abd2f3ec3c78b40d57714a27dadd50db/cf1: appending metadata at 1732454864335 (+16 ms)Flushing abd2f3ec3c78b40d57714a27dadd50db/cf1: closing flushed file at 1732454864335Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ce992df: reopening flushed file at 1732454864354 (+19 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for abd2f3ec3c78b40d57714a27dadd50db in 45ms, sequenceid=5, compaction requested=false at 1732454864362 (+8 ms)Writing region close event to WAL at 1732454864364 (+2 ms)Running coprocessor post-close hooks at 1732454864368 (+4 ms)Closed at 1732454864370 (+2 ms) 2024-11-24T13:27:44,371 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding abd2f3ec3c78b40d57714a27dadd50db move to b10924491aa5,38813,1732454849031 record at close sequenceid=5 2024-11-24T13:27:44,374 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,375 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=CLOSED 2024-11-24T13:27:44,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133 because future has completed 2024-11-24T13:27:44,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T13:27:44,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133 in 222 msec 2024-11-24T13:27:44,382 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE; state=CLOSED, location=b10924491aa5,38813,1732454849031; forceNewPlan=false, retain=false 2024-11-24T13:27:44,533 INFO [b10924491aa5:38765 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T13:27:44,533 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPENING, regionLocation=b10924491aa5,38813,1732454849031 2024-11-24T13:27:44,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE because future has completed 2024-11-24T13:27:44,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,38813,1732454849031}] 2024-11-24T13:27:44,694 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,694 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:44,694 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,694 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:44,695 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,695 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,696 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,697 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf1 2024-11-24T13:27:44,697 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:44,704 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77 2024-11-24T13:27:44,704 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:44,704 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,706 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf2 2024-11-24T13:27:44,706 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:44,706 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:44,706 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,731 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,733 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,734 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,734 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,735 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T13:27:44,736 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,737 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened abd2f3ec3c78b40d57714a27dadd50db; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58911391, jitterRate=-0.12215186655521393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T13:27:44,737 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:44,738 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for abd2f3ec3c78b40d57714a27dadd50db: Running coprocessor pre-open hook at 1732454864695Writing region info on filesystem at 1732454864695Initializing all the Stores at 1732454864696 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454864696Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454864696Cleaning up temporary data from old regions at 1732454864734 (+38 ms)Running coprocessor post-open hooks at 1732454864737 (+3 ms)Region opened successfully at 1732454864737 2024-11-24T13:27:44,739 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., pid=9, masterSystemTime=1732454864689 2024-11-24T13:27:44,742 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,742 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPEN, openSeqNum=9, regionLocation=b10924491aa5,38813,1732454849031 2024-11-24T13:27:44,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,38813,1732454849031 because future has completed 2024-11-24T13:27:44,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T13:27:44,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,38813,1732454849031 in 211 msec 2024-11-24T13:27:44,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE in 600 msec 2024-11-24T13:27:44,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:27:44,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:27:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:59630 deadline: 1732454924761, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=38813 startCode=1732454849031. As of locationSeqNum=5. 2024-11-24T13:27:44,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=38813 startCode=1732454849031. As of locationSeqNum=5. 2024-11-24T13:27:44,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=38813 startCode=1732454849031. As of locationSeqNum=5. 2024-11-24T13:27:44,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,38813,1732454849031, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=38813 startCode=1732454849031. As of locationSeqNum=5. 2024-11-24T13:27:44,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:27:44,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:27:44,890 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing abd2f3ec3c78b40d57714a27dadd50db 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T13:27:44,907 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/6ba20897edf6438eacf643c6d3483ce1 is 29, key is r1/cf1:/1732454864881/DeleteFamily/seqid=0 2024-11-24T13:27:44,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741888_1066 (size=4906) 2024-11-24T13:27:44,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741888_1066 (size=4906) 2024-11-24T13:27:44,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741888_1066 (size=4906) 2024-11-24T13:27:44,916 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:44,922 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:44,937 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf2/9b24feb1577b4e768a5d1f950483b9b6 is 29, key is r1/cf2:/1732454864881/DeleteFamily/seqid=0 2024-11-24T13:27:44,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741889_1067 (size=4906) 2024-11-24T13:27:44,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741889_1067 (size=4906) 2024-11-24T13:27:44,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741889_1067 (size=4906) 2024-11-24T13:27:44,944 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf2/9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:44,950 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:44,951 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/6ba20897edf6438eacf643c6d3483ce1 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:44,956 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:44,957 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T13:27:44,958 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf2/9b24feb1577b4e768a5d1f950483b9b6 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:44,963 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:44,963 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T13:27:44,964 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for abd2f3ec3c78b40d57714a27dadd50db in 74ms, sequenceid=12, compaction requested=false 2024-11-24T13:27:44,964 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for abd2f3ec3c78b40d57714a27dadd50db: 2024-11-24T13:27:44,967 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T13:27:44,968 DEBUG [Time-limited test {}] regionserver.HStore(1541): abd2f3ec3c78b40d57714a27dadd50db/cf1 is initiating major compaction (all files) 2024-11-24T13:27:44,968 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:27:44,969 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:44,969 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of abd2f3ec3c78b40d57714a27dadd50db/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:44,970 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77, hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1] into tmpdir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp, totalSize=9.5 K 2024-11-24T13:27:44,971 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 53b2858e0df6445fbbbc4f06125cea77, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732454864132 2024-11-24T13:27:44,971 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6ba20897edf6438eacf643c6d3483ce1, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T13:27:44,981 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): abd2f3ec3c78b40d57714a27dadd50db#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T13:27:44,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741890_1068 (size=4626) 2024-11-24T13:27:44,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741890_1068 (size=4626) 2024-11-24T13:27:44,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741890_1068 (size=4626) 2024-11-24T13:27:44,995 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf1/d769facfb2c340dea467ebfd34fd3fe4 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/d769facfb2c340dea467ebfd34fd3fe4 2024-11-24T13:27:45,007 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in abd2f3ec3c78b40d57714a27dadd50db/cf1 of abd2f3ec3c78b40d57714a27dadd50db into d769facfb2c340dea467ebfd34fd3fe4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T13:27:45,008 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for abd2f3ec3c78b40d57714a27dadd50db: 2024-11-24T13:27:45,008 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T13:27:45,008 DEBUG [Time-limited test {}] regionserver.HStore(1541): abd2f3ec3c78b40d57714a27dadd50db/cf2 is initiating major compaction (all files) 2024-11-24T13:27:45,008 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:27:45,008 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:27:45,008 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of abd2f3ec3c78b40d57714a27dadd50db/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,008 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6] into tmpdir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp, totalSize=4.8 K 2024-11-24T13:27:45,009 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9b24feb1577b4e768a5d1f950483b9b6, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T13:27:45,014 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): abd2f3ec3c78b40d57714a27dadd50db#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T13:27:45,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741891_1069 (size=4592) 2024-11-24T13:27:45,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741891_1069 (size=4592) 2024-11-24T13:27:45,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741891_1069 (size=4592) 2024-11-24T13:27:45,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/.tmp/cf2/74f253382fcd4fc2b02a4c502cadcd29 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/74f253382fcd4fc2b02a4c502cadcd29 2024-11-24T13:27:45,036 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in abd2f3ec3c78b40d57714a27dadd50db/cf2 of abd2f3ec3c78b40d57714a27dadd50db into 74f253382fcd4fc2b02a4c502cadcd29(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T13:27:45,036 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for abd2f3ec3c78b40d57714a27dadd50db: 2024-11-24T13:27:45,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=abd2f3ec3c78b40d57714a27dadd50db, source=b10924491aa5,38813,1732454849031, destination=b10924491aa5,34273,1732454849133, warming up region on b10924491aa5,34273,1732454849133 2024-11-24T13:27:45,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=abd2f3ec3c78b40d57714a27dadd50db, source=b10924491aa5,38813,1732454849031, destination=b10924491aa5,34273,1732454849133, running balancer 2024-11-24T13:27:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE 2024-11-24T13:27:45,042 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE 2024-11-24T13:27:45,044 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=CLOSING, regionLocation=b10924491aa5,38813,1732454849031 2024-11-24T13:27:45,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(7855): Warmup {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:45,045 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,046 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf1 2024-11-24T13:27:45,046 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:45,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE because future has completed 2024-11-24T13:27:45,046 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T13:27:45,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,38813,1732454849031}] 2024-11-24T13:27:45,053 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77 2024-11-24T13:27:45,058 INFO [StoreFileOpener-abd2f3ec3c78b40d57714a27dadd50db-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:45,058 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:45,062 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/d769facfb2c340dea467ebfd34fd3fe4 2024-11-24T13:27:45,062 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:45,062 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,063 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf2 2024-11-24T13:27:45,063 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:45,069 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/74f253382fcd4fc2b02a4c502cadcd29 2024-11-24T13:27:45,074 INFO [StoreFileOpener-abd2f3ec3c78b40d57714a27dadd50db-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:45,074 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:45,074 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:27:45,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:27:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34273 {}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454865074Disabling compacts and flushes for region at 1732454865074Disabling writes for close at 1732454865075 (+1 ms)Writing region close event to WAL at 1732454865076 (+1 ms)Closed at 1732454865076 2024-11-24T13:27:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T13:27:45,200 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,200 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T13:27:45,201 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:27:45,201 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,201 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,201 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:27:45,201 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,201 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77, hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1] to archive 2024-11-24T13:27:45,205 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T13:27:45,208 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77 to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/53b2858e0df6445fbbbc4f06125cea77 2024-11-24T13:27:45,210 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1 to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/6ba20897edf6438eacf643c6d3483ce1 2024-11-24T13:27:45,223 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6] to archive 2024-11-24T13:27:45,224 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T13:27:45,226 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6 to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/9b24feb1577b4e768a5d1f950483b9b6 2024-11-24T13:27:45,231 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T13:27:45,231 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,231 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454865200Running coprocessor pre-close hooks at 1732454865200Disabling compacts and flushes for region at 1732454865200Disabling writes for close at 1732454865201 (+1 ms)Writing region close event to WAL at 1732454865227 (+26 ms)Running coprocessor post-close hooks at 1732454865231 (+4 ms)Closed at 1732454865231 2024-11-24T13:27:45,231 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding abd2f3ec3c78b40d57714a27dadd50db move to b10924491aa5,34273,1732454849133 record at close sequenceid=12 2024-11-24T13:27:45,234 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=CLOSED 2024-11-24T13:27:45,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,38813,1732454849031 because future has completed 2024-11-24T13:27:45,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T13:27:45,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,38813,1732454849031 in 192 msec 2024-11-24T13:27:45,242 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE; state=CLOSED, location=b10924491aa5,34273,1732454849133; forceNewPlan=false, retain=false 2024-11-24T13:27:45,393 INFO [b10924491aa5:38765 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T13:27:45,393 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPENING, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:45,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE because future has completed 2024-11-24T13:27:45,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133}] 2024-11-24T13:27:45,554 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,554 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:45,554 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,554 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:45,554 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,554 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,556 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,557 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf1 2024-11-24T13:27:45,557 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:45,563 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/d769facfb2c340dea467ebfd34fd3fe4 2024-11-24T13:27:45,563 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:45,563 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,565 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf2 2024-11-24T13:27:45,565 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:45,571 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/74f253382fcd4fc2b02a4c502cadcd29 2024-11-24T13:27:45,571 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:45,571 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,572 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,573 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,574 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,574 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,575 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T13:27:45,576 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,577 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened abd2f3ec3c78b40d57714a27dadd50db; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67850156, jitterRate=0.011046111583709717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T13:27:45,577 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,577 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for abd2f3ec3c78b40d57714a27dadd50db: Running coprocessor pre-open hook at 1732454865555Writing region info on filesystem at 1732454865555Initializing all the Stores at 1732454865555Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454865555Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454865556 (+1 ms)Cleaning up temporary data from old regions at 1732454865574 (+18 ms)Running coprocessor post-open hooks at 1732454865577 (+3 ms)Region opened successfully at 1732454865577 2024-11-24T13:27:45,578 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., pid=12, masterSystemTime=1732454865550 2024-11-24T13:27:45,581 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,581 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,581 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPEN, openSeqNum=18, regionLocation=b10924491aa5,34273,1732454849133 2024-11-24T13:27:45,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133 because future has completed 2024-11-24T13:27:45,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T13:27:45,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,34273,1732454849133 in 188 msec 2024-11-24T13:27:45,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, REOPEN/MOVE in 548 msec 2024-11-24T13:27:45,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:27:45,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:27:45,597 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b10924491aa5,34273,1732454849133: testing ***** 2024-11-24T13:27:45,597 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T13:27:45,599 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T13:27:45,600 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T13:27:45,602 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T13:27:45,603 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T13:27:45,610 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 219502784 }, "NonHeapMemoryUsage": { "committed": 170721280, "init": 7667712, "max": -1, "used": 167975112 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b10924491aa5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b10924491aa5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49808", "tag.serverName": "b10924491aa5,45225,1732454849178", "tag.clusterId": "5f662c35-c445-40ae-a136-e68755f8700b", "tag.Context": "regionserver", "tag.Hostname": "b10924491aa5", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732454849178, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T13:27:45,613 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38765 {}] master.MasterRpcServices(700): b10924491aa5,34273,1732454849133 reported a fatal error: ***** ABORTING region server b10924491aa5,34273,1732454849133: testing ***** 2024-11-24T13:27:45,615 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b10924491aa5,34273,1732454849133' ***** 2024-11-24T13:27:45,615 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T13:27:45,616 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:27:45,616 INFO [RS:1;b10924491aa5:34273 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T13:27:45,616 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:27:45,616 INFO [RS:1;b10924491aa5:34273 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T13:27:45,616 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(3091): Received CLOSE for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:52674 deadline: 1732454925616, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=34273 startCode=1732454849133. As of locationSeqNum=12. 2024-11-24T13:27:45,617 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(956): aborting server b10924491aa5,34273,1732454849133 2024-11-24T13:27:45,617 INFO [RS:1;b10924491aa5:34273 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:27:45,617 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,38813,1732454849031, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,38813,1732454849031, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=34273 startCode=1732454849133. As of locationSeqNum=12. 2024-11-24T13:27:45,617 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:27:45,617 INFO [RS:1;b10924491aa5:34273 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b10924491aa5:34273. 2024-11-24T13:27:45,617 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,38813,1732454849031, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=34273 startCode=1732454849133. As of locationSeqNum=12. 2024-11-24T13:27:45,617 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,617 DEBUG [RS:1;b10924491aa5:34273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:27:45,617 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,38813,1732454849031, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b10924491aa5 port=34273 startCode=1732454849133. As of locationSeqNum=12. 2024-11-24T13:27:45,617 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,617 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:27:45,618 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,618 DEBUG [RS:1;b10924491aa5:34273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:27:45,618 INFO [RS:1;b10924491aa5:34273 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:27:45,618 INFO [RS:1;b10924491aa5:34273 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:27:45,618 INFO [RS:1;b10924491aa5:34273 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:27:45,618 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T13:27:45,619 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T13:27:45,619 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, abd2f3ec3c78b40d57714a27dadd50db=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.} 2024-11-24T13:27:45,619 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:27:45,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b10924491aa5,34273,1732454849133 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:45,619 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:27:45,619 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:27:45,619 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:27:45,619 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:27:45,620 DEBUG [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:45,620 ERROR [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-24T13:27:45,620 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:27:45,620 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,620 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:27:45,620 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454865617Running coprocessor pre-close hooks at 1732454865617Disabling compacts and flushes for region at 1732454865617Disabling writes for close at 1732454865618 (+1 ms)Writing region close event to WAL at 1732454865620 (+2 ms)Running coprocessor post-close hooks at 1732454865620Closed at 1732454865620 2024-11-24T13:27:45,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b10924491aa5,34273,1732454849133 aborting 2024-11-24T13:27:45,621 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732454865619Running coprocessor pre-close hooks at 1732454865619Disabling compacts and flushes for region at 1732454865619Disabling writes for close at 1732454865619Writing region close event to WAL at 1732454865620 (+1 ms)Running coprocessor post-close hooks at 1732454865620Closed at 1732454865620 2024-11-24T13:27:45,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b10924491aa5,34273,1732454849133 aborting 2024-11-24T13:27:45,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 from cache 2024-11-24T13:27:45,621 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:45,621 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T13:27:45,650 INFO [regionserver/b10924491aa5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:27:45,727 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:45,729 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1] 2024-11-24T13:27:45,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b10924491aa5,34273,1732454849133 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:45,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b10924491aa5,34273,1732454849133 aborting 2024-11-24T13:27:45,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b10924491aa5,34273,1732454849133 aborting 2024-11-24T13:27:45,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 from cache 2024-11-24T13:27:45,820 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(976): stopping server b10924491aa5,34273,1732454849133; all regions closed. 2024-11-24T13:27:45,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741836_1012 (size=6845) 2024-11-24T13:27:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741836_1012 (size=6845) 2024-11-24T13:27:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741836_1012 (size=6845) 2024-11-24T13:27:45,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741835_1011 (size=2142) 2024-11-24T13:27:45,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741835_1011 (size=2142) 2024-11-24T13:27:45,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741835_1011 (size=2142) 2024-11-24T13:27:45,828 DEBUG [RS:1;b10924491aa5:34273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:27:45,828 INFO [RS:1;b10924491aa5:34273 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:27:45,828 INFO [RS:1;b10924491aa5:34273 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:27:45,829 INFO [RS:1;b10924491aa5:34273 {}] hbase.ChoreService(370): Chore service for: regionserver/b10924491aa5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T13:27:45,829 INFO [RS:1;b10924491aa5:34273 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:27:45,829 INFO [regionserver/b10924491aa5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:27:45,829 INFO [RS:1;b10924491aa5:34273 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34273 2024-11-24T13:27:45,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b10924491aa5,34273,1732454849133 2024-11-24T13:27:45,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:27:45,837 INFO [RS:1;b10924491aa5:34273 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:27:45,838 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b10924491aa5,34273,1732454849133] 2024-11-24T13:27:45,839 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b10924491aa5,34273,1732454849133 already deleted, retry=false 2024-11-24T13:27:45,840 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b10924491aa5,34273,1732454849133 on b10924491aa5,38765,1732454848333 2024-11-24T13:27:45,845 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b10924491aa5,34273,1732454849133, splitWal=true, meta=true 2024-11-24T13:27:45,847 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for b10924491aa5,34273,1732454849133 (carryingMeta=true) b10924491aa5,34273,1732454849133/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4af3499f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T13:27:45,847 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b10924491aa5,34273,1732454849133, splitWal=true, meta=true 2024-11-24T13:27:45,849 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure b10924491aa5,34273,1732454849133, splitWal=true, meta=true, isMeta: true 2024-11-24T13:27:45,850 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting 2024-11-24T13:27:45,851 INFO [PEWorker-1 {}] master.SplitWALManager(105): b10924491aa5,34273,1732454849133 WAL count=1, meta=true 2024-11-24T13:27:45,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta}] 2024-11-24T13:27:45,859 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=b10924491aa5,38813,1732454849031 2024-11-24T13:27:45,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, worker=b10924491aa5,38813,1732454849031}] 2024-11-24T13:27:45,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:45,939 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1] 2024-11-24T13:27:45,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:27:45,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34273-0x1014a28febd0002, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:27:45,940 INFO [RS:1;b10924491aa5:34273 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:27:45,940 INFO [RS:1;b10924491aa5:34273 {}] regionserver.HRegionServer(1031): Exiting; stopping=b10924491aa5,34273,1732454849133; zookeeper connection closed. 2024-11-24T13:27:45,941 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74f641da {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74f641da 2024-11-24T13:27:45,941 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b10924491aa5:34273 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b10924491aa5/172.17.0.2:34273 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:45,942 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1, error=java.net.ConnectException: Call to address=b10924491aa5:34273 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b10924491aa5/172.17.0.2:34273 2024-11-24T13:27:45,942 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-24T13:27:45,942 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 from cache 2024-11-24T13:27:45,942 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address b10924491aa5:34273 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b10924491aa5/172.17.0.2:34273 2024-11-24T13:27:46,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T13:27:46,042 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, size=6.7 K (6845bytes) 2024-11-24T13:27:46,042 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta 2024-11-24T13:27:46,043 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta after 1ms 2024-11-24T13:27:46,046 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta took 4ms 2024-11-24T13:27:46,054 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T13:27:46,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta so closing down 2024-11-24T13:27:46,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:46,056 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp 2024-11-24T13:27:46,057 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp 2024-11-24T13:27:46,058 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:46,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741892_1070 (size=6604) 2024-11-24T13:27:46,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741892_1070 (size=6604) 2024-11-24T13:27:46,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741892_1070 (size=6604) 2024-11-24T13:27:46,066 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:46,067 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T13:27:46,068 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-24T13:27:46,068 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, journal: Splitting hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, size=6.7 K (6845bytes) at 1732454866042Finishing writing output for hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta so closing down at 1732454866055 (+13 ms)Creating recovered edits writer path=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp at 1732454866057 (+2 ms)3 split writer threads finished at 1732454866058 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732454866066 (+8 ms)Rename recovered edits hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta.temp to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732454866067 (+1 ms)Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1732454866068 (+1 ms) 2024-11-24T13:27:46,068 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta 2024-11-24T13:27:46,069 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T13:27:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T13:27:46,076 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs 2024-11-24T13:27:46,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T13:27:46,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, worker=b10924491aa5,38813,1732454849031 in 215 msec 2024-11-24T13:27:46,080 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=b10924491aa5,38813,1732454849031 2024-11-24T13:27:46,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T13:27:46,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b10924491aa5%2C34273%2C1732454849133.meta.1732454850903.meta, worker=b10924491aa5,38813,1732454849031 in 226 msec 2024-11-24T13:27:46,084 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting dir is empty, no logs to split. 2024-11-24T13:27:46,084 INFO [PEWorker-1 {}] master.SplitWALManager(105): b10924491aa5,34273,1732454849133 WAL count=0, meta=true 2024-11-24T13:27:46,084 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b10924491aa5,34273,1732454849133 WAL splitting is done? wals=0, meta=true 2024-11-24T13:27:46,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T13:27:46,087 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:27:46,088 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T13:27:46,238 DEBUG [b10924491aa5:38765 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T13:27:46,239 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(204): Hosts are {b10924491aa5=0} racks are {/default-rack=0} 2024-11-24T13:27:46,239 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:27:46,239 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:27:46,239 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:27:46,239 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:27:46,239 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:27:46,239 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:27:46,239 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:27:46,239 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b10924491aa5,38813,1732454849031 2024-11-24T13:27:46,241 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b10924491aa5,38813,1732454849031, state=OPENING 2024-11-24T13:27:46,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:46,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:46,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:46,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:46,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:46,244 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:27:46,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:46,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b10924491aa5,38813,1732454849031}] 2024-11-24T13:27:46,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:46,248 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1] 2024-11-24T13:27:46,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:52674 deadline: 1732454926249, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on b10924491aa5,38813,1732454849031 2024-11-24T13:27:46,250 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on b10924491aa5,38813,1732454849031 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T13:27:46,250 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on b10924491aa5,38813,1732454849031 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T13:27:46,250 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1 from cache 2024-11-24T13:27:46,400 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T13:27:46,400 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:46,401 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T13:27:46,402 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b10924491aa5%2C38813%2C1732454849031.meta, suffix=.meta, logDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031, archiveDir=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs, maxLogs=32 2024-11-24T13:27:46,416 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031/b10924491aa5%2C38813%2C1732454849031.meta.1732454866403.meta, exclude list is [], retry=0 2024-11-24T13:27:46,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:46,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:46,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:46,420 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031/b10924491aa5%2C38813%2C1732454849031.meta.1732454866403.meta 2024-11-24T13:27:46,421 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:46,421 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:46,421 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T13:27:46,421 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T13:27:46,422 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T13:27:46,422 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T13:27:46,422 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:46,422 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T13:27:46,422 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T13:27:46,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T13:27:46,425 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:27:46,425 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:46,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:46,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T13:27:46,427 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:27:46,427 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:46,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:46,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T13:27:46,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:27:46,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:46,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:46,429 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T13:27:46,429 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:27:46,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:46,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:27:46,430 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:27:46,430 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740 2024-11-24T13:27:46,432 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740 2024-11-24T13:27:46,432 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T13:27:46,436 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T13:27:46,436 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-24T13:27:46,456 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/info/f16107b44ecb48fdb672279c7ffde503 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db./info:regioninfo/1732454865581/Put/seqid=0 2024-11-24T13:27:46,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741894_1072 (size=11177) 2024-11-24T13:27:46,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741894_1072 (size=11177) 2024-11-24T13:27:46,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741894_1072 (size=11177) 2024-11-24T13:27:46,465 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/info/f16107b44ecb48fdb672279c7ffde503 2024-11-24T13:27:46,485 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/ns/961f010de92944b98e265f1615506e66 is 43, key is default/ns:d/1732454851054/Put/seqid=0 2024-11-24T13:27:46,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741895_1073 (size=5153) 2024-11-24T13:27:46,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741895_1073 (size=5153) 2024-11-24T13:27:46,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741895_1073 (size=5153) 2024-11-24T13:27:46,492 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/ns/961f010de92944b98e265f1615506e66 2024-11-24T13:27:46,511 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/table/689c4088b7554607bb9cc4ef0713fd5b is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732454863884/Put/seqid=0 2024-11-24T13:27:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741896_1074 (size=5431) 2024-11-24T13:27:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741896_1074 (size=5431) 2024-11-24T13:27:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741896_1074 (size=5431) 2024-11-24T13:27:46,520 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/table/689c4088b7554607bb9cc4ef0713fd5b 2024-11-24T13:27:46,526 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/info/f16107b44ecb48fdb672279c7ffde503 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/info/f16107b44ecb48fdb672279c7ffde503 2024-11-24T13:27:46,533 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/info/f16107b44ecb48fdb672279c7ffde503, entries=36, sequenceid=18, filesize=10.9 K 2024-11-24T13:27:46,534 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/ns/961f010de92944b98e265f1615506e66 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/ns/961f010de92944b98e265f1615506e66 2024-11-24T13:27:46,540 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/ns/961f010de92944b98e265f1615506e66, entries=2, sequenceid=18, filesize=5.0 K 2024-11-24T13:27:46,541 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/table/689c4088b7554607bb9cc4ef0713fd5b as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/table/689c4088b7554607bb9cc4ef0713fd5b 2024-11-24T13:27:46,547 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/table/689c4088b7554607bb9cc4ef0713fd5b, entries=2, sequenceid=18, filesize=5.3 K 2024-11-24T13:27:46,547 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=18, compaction requested=false; wal=null 2024-11-24T13:27:46,548 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T13:27:46,549 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:27:46,549 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:27:46,550 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:27:46,552 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:27:46,554 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T13:27:46,555 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60106487, jitterRate=-0.10434354841709137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:27:46,556 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T13:27:46,556 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732454866422Writing region info on filesystem at 1732454866422Initializing all the Stores at 1732454866423 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454866423Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454866424 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454866424Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732454866424Obtaining lock to block concurrent updates at 1732454866437 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1732454866437Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732454866437Flushing stores of hbase:meta,,1.1588230740 at 1732454866437Flushing 1588230740/info: creating writer at 1732454866437Flushing 1588230740/info: appending metadata at 1732454866455 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732454866455Flushing 1588230740/ns: creating writer at 1732454866471 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732454866485 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732454866485Flushing 1588230740/table: creating writer at 1732454866497 (+12 ms)Flushing 1588230740/table: appending metadata at 1732454866511 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732454866511Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@612d73b1: reopening flushed file at 1732454866526 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69c3c321: reopening flushed file at 1732454866533 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50788a4e: reopening flushed file at 1732454866540 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=18, compaction requested=false; wal=null at 1732454866547 (+7 ms)Cleaning up temporary data from old regions at 1732454866550 (+3 ms)Running coprocessor post-open hooks at 1732454866556 (+6 ms)Region opened successfully at 1732454866556 2024-11-24T13:27:46,557 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732454866396 2024-11-24T13:27:46,559 DEBUG [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T13:27:46,560 INFO [RS_OPEN_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T13:27:46,560 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=b10924491aa5,38813,1732454849031 2024-11-24T13:27:46,561 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b10924491aa5,38813,1732454849031, state=OPEN 2024-11-24T13:27:46,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:46,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:46,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:27:46,563 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b10924491aa5,38813,1732454849031 2024-11-24T13:27:46,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:46,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:46,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:27:46,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T13:27:46,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b10924491aa5,38813,1732454849031 in 319 msec 2024-11-24T13:27:46,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T13:27:46,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 480 msec 2024-11-24T13:27:46,569 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): b10924491aa5,34273,1732454849133 had 2 regions 2024-11-24T13:27:46,570 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b10924491aa5,34273,1732454849133, splitWal=true, meta=true, isMeta: false 2024-11-24T13:27:46,572 INFO [PEWorker-5 {}] master.SplitWALManager(105): b10924491aa5,34273,1732454849133 WAL count=1, meta=false 2024-11-24T13:27:46,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b10924491aa5%2C34273%2C1732454849133.1732454850477}] 2024-11-24T13:27:46,574 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=b10924491aa5,45225,1732454849178 2024-11-24T13:27:46,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b10924491aa5%2C34273%2C1732454849133.1732454850477, worker=b10924491aa5,45225,1732454849178}] 2024-11-24T13:27:46,727 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:27:46,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43903, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:27:46,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45225 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-24T13:27:46,746 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477, size=2.1 K (2142bytes) 2024-11-24T13:27:46,746 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 2024-11-24T13:27:46,746 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 after 0ms 2024-11-24T13:27:46,749 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 took 3ms 2024-11-24T13:27:46,751 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for abd2f3ec3c78b40d57714a27dadd50db: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T13:27:46,751 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 so closing down 2024-11-24T13:27:46,751 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:46,751 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:46,751 INFO [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-24T13:27:46,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477, journal: Splitting hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477, size=2.1 K (2142bytes) at 1732454866746Finishing writing output for hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 so closing down at 1732454866751 (+5 ms)3 split writer threads finished at 1732454866751Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732454866751 2024-11-24T13:27:46,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 2024-11-24T13:27:46,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b10924491aa5:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-24T13:27:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38765 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-24T13:27:46,756 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting/b10924491aa5%2C34273%2C1732454849133.1732454850477 to hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs 2024-11-24T13:27:46,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:46,758 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1] 2024-11-24T13:27:46,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-24T13:27:46,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b10924491aa5%2C34273%2C1732454849133.1732454850477, worker=b10924491aa5,45225,1732454849178 in 181 msec 2024-11-24T13:27:46,760 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b10924491aa5,45225,1732454849178 2024-11-24T13:27:46,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-24T13:27:46,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b10924491aa5%2C34273%2C1732454849133.1732454850477, worker=b10924491aa5,45225,1732454849178 in 188 msec 2024-11-24T13:27:46,764 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,34273,1732454849133-splitting dir is empty, no logs to split. 2024-11-24T13:27:46,764 INFO [PEWorker-5 {}] master.SplitWALManager(105): b10924491aa5,34273,1732454849133 WAL count=0, meta=false 2024-11-24T13:27:46,764 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if b10924491aa5,34273,1732454849133 WAL splitting is done? wals=0, meta=false 2024-11-24T13:27:46,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN}] 2024-11-24T13:27:46,767 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN 2024-11-24T13:27:46,768 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T13:27:46,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to b10924491aa5:34273 this server is in the failed servers list 2024-11-24T13:27:46,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b10924491aa5:34273 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b10924491aa5:34273 2024-11-24T13:27:46,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b10924491aa5:34273 2024-11-24T13:27:46,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=12 from cache 2024-11-24T13:27:46,919 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(204): Hosts are {b10924491aa5=0} racks are {/default-rack=0} 2024-11-24T13:27:46,919 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:27:46,919 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:27:46,919 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:27:46,919 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:27:46,919 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:27:46,919 INFO [b10924491aa5:38765 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:27:46,919 DEBUG [b10924491aa5:38765 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:27:46,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPENING, regionLocation=b10924491aa5,45225,1732454849178 2024-11-24T13:27:46,920 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b10924491aa5:34273 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b10924491aa5/172.17.0.2:34273 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:46,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1, error=java.net.ConnectException: Call to address=b10924491aa5:34273 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b10924491aa5/172.17.0.2:34273 2024-11-24T13:27:46,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T13:27:46,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b10924491aa5,34273,1732454849133, seqNum=-1 from cache 2024-11-24T13:27:46,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b10924491aa5:34273 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b10924491aa5/172.17.0.2:34273 2024-11-24T13:27:47,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:27:47,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b10924491aa5,38813,1732454849031, seqNum=-1] 2024-11-24T13:27:47,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:27:47,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:27:47,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN because future has completed 2024-11-24T13:27:47,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,45225,1732454849178}] 2024-11-24T13:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=18] 2024-11-24T13:27:47,093 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to b10924491aa5:34273 this server is in the failed servers list 2024-11-24T13:27:47,094 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b10924491aa5:34273 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b10924491aa5:34273 2024-11-24T13:27:47,094 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b10924491aa5:34273 2024-11-24T13:27:47,094 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,34273,1732454849133, seqNum=18 from cache 2024-11-24T13:27:47,188 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:47,188 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => abd2f3ec3c78b40d57714a27dadd50db, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:47,188 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,188 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:47,188 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,188 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,189 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,190 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf1 2024-11-24T13:27:47,190 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,196 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf1/d769facfb2c340dea467ebfd34fd3fe4 2024-11-24T13:27:47,196 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,196 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,197 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abd2f3ec3c78b40d57714a27dadd50db columnFamilyName cf2 2024-11-24T13:27:47,197 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,202 DEBUG [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/cf2/74f253382fcd4fc2b02a4c502cadcd29 2024-11-24T13:27:47,203 INFO [StoreOpener-abd2f3ec3c78b40d57714a27dadd50db-1 {}] regionserver.HStore(327): Store=abd2f3ec3c78b40d57714a27dadd50db/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,203 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,203 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,204 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,205 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,205 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,206 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T13:27:47,207 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,207 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened abd2f3ec3c78b40d57714a27dadd50db; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61820341, jitterRate=-0.07880513370037079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T13:27:47,208 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:27:47,208 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for abd2f3ec3c78b40d57714a27dadd50db: Running coprocessor pre-open hook at 1732454867188Writing region info on filesystem at 1732454867188Initializing all the Stores at 1732454867189 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867189Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867189Cleaning up temporary data from old regions at 1732454867205 (+16 ms)Running coprocessor post-open hooks at 1732454867208 (+3 ms)Region opened successfully at 1732454867208 2024-11-24T13:27:47,209 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., pid=21, masterSystemTime=1732454867183 2024-11-24T13:27:47,211 DEBUG [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:47,211 INFO [RS_OPEN_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:27:47,212 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=abd2f3ec3c78b40d57714a27dadd50db, regionState=OPEN, openSeqNum=18, regionLocation=b10924491aa5,45225,1732454849178 2024-11-24T13:27:47,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,45225,1732454849178 because future has completed 2024-11-24T13:27:47,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T13:27:47,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure abd2f3ec3c78b40d57714a27dadd50db, server=b10924491aa5,45225,1732454849178 in 184 msec 2024-11-24T13:27:47,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-24T13:27:47,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=abd2f3ec3c78b40d57714a27dadd50db, ASSIGN in 452 msec 2024-11-24T13:27:47,220 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server b10924491aa5,34273,1732454849133 after splitting done 2024-11-24T13:27:47,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure b10924491aa5,34273,1732454849133, splitWal=true, meta=true in 1.3800 sec 2024-11-24T13:27:47,401 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db., hostname=b10924491aa5,45225,1732454849178, seqNum=18] 2024-11-24T13:27:47,401 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:27:47,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:27:47,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412 (was 415), OpenFileDescriptor=1050 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=7087 (was 7223) 2024-11-24T13:27:47,419 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-24T13:27:47,431 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=412, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=7087 2024-11-24T13:27:47,431 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-24T13:27:47,446 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:47,448 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:47,448 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:47,451 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30722921, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-30722921, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:47,464 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30722921/hregion-30722921.1732454867451, exclude list is [], retry=0 2024-11-24T13:27:47,466 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:47,466 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:47,468 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:47,469 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30722921/hregion-30722921.1732454867451 2024-11-24T13:27:47,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:47,470 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 45fbe296a2b4cc55cced075d9faec7db, NAME => 'testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:47,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741898_1076 (size=67) 2024-11-24T13:27:47,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741898_1076 (size=67) 2024-11-24T13:27:47,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741898_1076 (size=67) 2024-11-24T13:27:47,481 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:47,482 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,483 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName a 2024-11-24T13:27:47,483 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,484 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,484 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,485 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName b 2024-11-24T13:27:47,485 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,485 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,486 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,486 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName c 2024-11-24T13:27:47,486 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,487 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,487 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,488 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,488 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,489 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,489 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,489 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:47,490 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,492 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:47,493 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45fbe296a2b4cc55cced075d9faec7db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70842044, jitterRate=0.05562871694564819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:47,493 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45fbe296a2b4cc55cced075d9faec7db: Writing region info on filesystem at 1732454867481Initializing all the Stores at 1732454867482 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867482Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867482Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867482Cleaning up temporary data from old regions at 1732454867489 (+7 ms)Region opened successfully at 1732454867493 (+4 ms) 2024-11-24T13:27:47,493 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45fbe296a2b4cc55cced075d9faec7db, disabling compactions & flushes 2024-11-24T13:27:47,493 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,493 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,493 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. after waiting 0 ms 2024-11-24T13:27:47,493 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,494 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,494 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45fbe296a2b4cc55cced075d9faec7db: Waiting for close lock at 1732454867493Disabling compacts and flushes for region at 1732454867493Disabling writes for close at 1732454867493Writing region close event to WAL at 1732454867494 (+1 ms)Closed at 1732454867494 2024-11-24T13:27:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741897_1075 (size=93) 2024-11-24T13:27:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741897_1075 (size=93) 2024-11-24T13:27:47,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741897_1075 (size=93) 2024-11-24T13:27:47,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:47,499 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30722921:(num 1732454867451) 2024-11-24T13:27:47,499 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:47,500 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:47,514 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501, exclude list is [], retry=0 2024-11-24T13:27:47,516 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:47,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:47,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:47,519 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 2024-11-24T13:27:47,519 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:47,519 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 45fbe296a2b4cc55cced075d9faec7db, NAME => 'testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:47,519 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:47,519 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,519 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,521 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,522 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName a 2024-11-24T13:27:47,522 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,522 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,523 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,523 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName b 2024-11-24T13:27:47,523 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,524 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,524 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,524 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName c 2024-11-24T13:27:47,525 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,525 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,525 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,527 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,527 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,528 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:47,529 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,530 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45fbe296a2b4cc55cced075d9faec7db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59641208, jitterRate=-0.11127674579620361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:47,530 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45fbe296a2b4cc55cced075d9faec7db: Writing region info on filesystem at 1732454867519Initializing all the Stores at 1732454867520 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867520Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867520Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867520Cleaning up temporary data from old regions at 1732454867527 (+7 ms)Region opened successfully at 1732454867530 (+3 ms) 2024-11-24T13:27:47,554 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45fbe296a2b4cc55cced075d9faec7db 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T13:27:47,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/a/41761d36099b43d2a929cf70727f165b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732454867530/Put/seqid=0 2024-11-24T13:27:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741900_1078 (size=5958) 2024-11-24T13:27:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741900_1078 (size=5958) 2024-11-24T13:27:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741900_1078 (size=5958) 2024-11-24T13:27:47,578 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/a/41761d36099b43d2a929cf70727f165b 2024-11-24T13:27:47,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/b/6aa535fca0db4aaba4d4c43c5ca12bd2 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732454867538/Put/seqid=0 2024-11-24T13:27:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741901_1079 (size=5958) 2024-11-24T13:27:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741901_1079 (size=5958) 2024-11-24T13:27:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741901_1079 (size=5958) 2024-11-24T13:27:47,605 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/b/6aa535fca0db4aaba4d4c43c5ca12bd2 2024-11-24T13:27:47,624 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/c/e86f60b25841458ab97686c81f7dccc7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732454867546/Put/seqid=0 2024-11-24T13:27:47,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741902_1080 (size=5958) 2024-11-24T13:27:47,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741902_1080 (size=5958) 2024-11-24T13:27:47,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741902_1080 (size=5958) 2024-11-24T13:27:47,635 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/c/e86f60b25841458ab97686c81f7dccc7 2024-11-24T13:27:47,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/a/41761d36099b43d2a929cf70727f165b as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/a/41761d36099b43d2a929cf70727f165b 2024-11-24T13:27:47,647 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/a/41761d36099b43d2a929cf70727f165b, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T13:27:47,648 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/b/6aa535fca0db4aaba4d4c43c5ca12bd2 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/b/6aa535fca0db4aaba4d4c43c5ca12bd2 2024-11-24T13:27:47,653 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/b/6aa535fca0db4aaba4d4c43c5ca12bd2, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T13:27:47,653 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/c/e86f60b25841458ab97686c81f7dccc7 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/c/e86f60b25841458ab97686c81f7dccc7 2024-11-24T13:27:47,658 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/c/e86f60b25841458ab97686c81f7dccc7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T13:27:47,660 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 45fbe296a2b4cc55cced075d9faec7db in 106ms, sequenceid=33, compaction requested=false 2024-11-24T13:27:47,660 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 45fbe296a2b4cc55cced075d9faec7db: 2024-11-24T13:27:47,660 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45fbe296a2b4cc55cced075d9faec7db, disabling compactions & flushes 2024-11-24T13:27:47,660 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,660 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,660 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. after waiting 0 ms 2024-11-24T13:27:47,660 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,662 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. 2024-11-24T13:27:47,662 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45fbe296a2b4cc55cced075d9faec7db: Waiting for close lock at 1732454867660Disabling compacts and flushes for region at 1732454867660Disabling writes for close at 1732454867660Writing region close event to WAL at 1732454867662 (+2 ms)Closed at 1732454867662 2024-11-24T13:27:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741899_1077 (size=5937) 2024-11-24T13:27:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741899_1077 (size=5937) 2024-11-24T13:27:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741899_1077 (size=5937) 2024-11-24T13:27:47,670 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/b/6aa535fca0db4aaba4d4c43c5ca12bd2 to hdfs://localhost:35337/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/b/6aa535fca0db4aaba4d4c43c5ca12bd2 2024-11-24T13:27:47,687 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501, size=5.8 K (5937bytes) 2024-11-24T13:27:47,687 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 2024-11-24T13:27:47,687 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 after 0ms 2024-11-24T13:27:47,690 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 took 3ms 2024-11-24T13:27:47,693 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 so closing down 2024-11-24T13:27:47,693 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:47,694 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732454867501.temp 2024-11-24T13:27:47,695 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000003-wal.1732454867501.temp 2024-11-24T13:27:47,695 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:47,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741903_1081 (size=5223) 2024-11-24T13:27:47,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741903_1081 (size=5223) 2024-11-24T13:27:47,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741903_1081 (size=5223) 2024-11-24T13:27:47,704 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000003-wal.1732454867501.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:47,705 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000003-wal.1732454867501.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000032 2024-11-24T13:27:47,705 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-24T13:27:47,705 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501, journal: Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501, size=5.8 K (5937bytes) at 1732454867687Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 so closing down at 1732454867693 (+6 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000003-wal.1732454867501.temp at 1732454867695 (+2 ms)3 split writer threads finished at 1732454867695Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000003-wal.1732454867501.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732454867704 (+9 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000003-wal.1732454867501.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000032 at 1732454867705 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732454867705 2024-11-24T13:27:47,707 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867501 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454867501 2024-11-24T13:27:47,708 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000032 2024-11-24T13:27:47,708 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:47,709 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:47,723 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867710, exclude list is [], retry=0 2024-11-24T13:27:47,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:47,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:47,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:47,730 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867710 2024-11-24T13:27:47,730 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:47,730 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 45fbe296a2b4cc55cced075d9faec7db, NAME => 'testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:47,730 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:47,730 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,730 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,732 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,733 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName a 2024-11-24T13:27:47,733 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,741 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/a/41761d36099b43d2a929cf70727f165b 2024-11-24T13:27:47,741 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,742 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,743 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName b 2024-11-24T13:27:47,743 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,743 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,743 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,744 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fbe296a2b4cc55cced075d9faec7db columnFamilyName c 2024-11-24T13:27:47,744 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,749 DEBUG [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/c/e86f60b25841458ab97686c81f7dccc7 2024-11-24T13:27:47,749 INFO [StoreOpener-45fbe296a2b4cc55cced075d9faec7db-1 {}] regionserver.HStore(327): Store=45fbe296a2b4cc55cced075d9faec7db/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,750 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,750 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,752 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,752 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000032 2024-11-24T13:27:47,755 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000032 2024-11-24T13:27:47,755 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45fbe296a2b4cc55cced075d9faec7db 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T13:27:47,770 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/b/b2dba2d24c554292a4e36f7985bd22a5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732454867538/Put/seqid=0 2024-11-24T13:27:47,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741905_1083 (size=5958) 2024-11-24T13:27:47,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741905_1083 (size=5958) 2024-11-24T13:27:47,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741905_1083 (size=5958) 2024-11-24T13:27:47,778 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/b/b2dba2d24c554292a4e36f7985bd22a5 2024-11-24T13:27:47,784 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/.tmp/b/b2dba2d24c554292a4e36f7985bd22a5 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/b/b2dba2d24c554292a4e36f7985bd22a5 2024-11-24T13:27:47,788 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/b/b2dba2d24c554292a4e36f7985bd22a5, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T13:27:47,789 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 45fbe296a2b4cc55cced075d9faec7db in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T13:27:47,789 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/0000000000000000032 2024-11-24T13:27:47,790 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,790 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,791 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:47,792 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45fbe296a2b4cc55cced075d9faec7db 2024-11-24T13:27:47,794 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/45fbe296a2b4cc55cced075d9faec7db/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T13:27:47,794 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45fbe296a2b4cc55cced075d9faec7db; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63525110, jitterRate=-0.053402096033096313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:47,795 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45fbe296a2b4cc55cced075d9faec7db: Writing region info on filesystem at 1732454867731Initializing all the Stores at 1732454867731Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867732 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867732Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867732Obtaining lock to block concurrent updates at 1732454867755 (+23 ms)Preparing flush snapshotting stores in 45fbe296a2b4cc55cced075d9faec7db at 1732454867755Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732454867755Flushing stores of testReplayEditsWrittenViaHRegion,,1732454867446.45fbe296a2b4cc55cced075d9faec7db. at 1732454867755Flushing 45fbe296a2b4cc55cced075d9faec7db/b: creating writer at 1732454867755Flushing 45fbe296a2b4cc55cced075d9faec7db/b: appending metadata at 1732454867769 (+14 ms)Flushing 45fbe296a2b4cc55cced075d9faec7db/b: closing flushed file at 1732454867770 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@154145e8: reopening flushed file at 1732454867783 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 45fbe296a2b4cc55cced075d9faec7db in 34ms, sequenceid=32, compaction requested=false; wal=null at 1732454867789 (+6 ms)Cleaning up temporary data from old regions at 1732454867790 (+1 ms)Region opened successfully at 1732454867795 (+5 ms) 2024-11-24T13:27:47,816 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 412) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:52272 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:43828 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:52108 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:42796 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:42634 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:44042 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 281) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7079 (was 7087) 2024-11-24T13:27:47,816 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-24T13:27:47,828 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=7078 2024-11-24T13:27:47,828 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-24T13:27:47,843 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:47,845 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:47,846 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:47,848 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16772109, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-16772109, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:47,860 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16772109/hregion-16772109.1732454867848, exclude list is [], retry=0 2024-11-24T13:27:47,863 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:47,863 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:47,863 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:47,865 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16772109/hregion-16772109.1732454867848 2024-11-24T13:27:47,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:27:47,866 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d4a9029308c597bb51bd23508d89c905, NAME => 'testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:47,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741907_1085 (size=68) 2024-11-24T13:27:47,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741907_1085 (size=68) 2024-11-24T13:27:47,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741907_1085 (size=68) 2024-11-24T13:27:47,882 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:47,884 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,886 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName a 2024-11-24T13:27:47,886 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,886 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,887 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,888 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName b 2024-11-24T13:27:47,888 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,889 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,889 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,890 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName c 2024-11-24T13:27:47,891 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,891 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,891 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,894 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:47,895 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,896 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:47,897 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d4a9029308c597bb51bd23508d89c905; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72538257, jitterRate=0.08090426027774811}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:47,897 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d4a9029308c597bb51bd23508d89c905: Writing region info on filesystem at 1732454867882Initializing all the Stores at 1732454867884 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867884Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867884Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867884Cleaning up temporary data from old regions at 1732454867893 (+9 ms)Region opened successfully at 1732454867897 (+4 ms) 2024-11-24T13:27:47,898 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d4a9029308c597bb51bd23508d89c905, disabling compactions & flushes 2024-11-24T13:27:47,898 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:47,898 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:47,898 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. after waiting 0 ms 2024-11-24T13:27:47,898 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:47,898 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:47,898 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d4a9029308c597bb51bd23508d89c905: Waiting for close lock at 1732454867897Disabling compacts and flushes for region at 1732454867897Disabling writes for close at 1732454867898 (+1 ms)Writing region close event to WAL at 1732454867898Closed at 1732454867898 2024-11-24T13:27:47,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741906_1084 (size=93) 2024-11-24T13:27:47,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741906_1084 (size=93) 2024-11-24T13:27:47,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:47,903 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16772109:(num 1732454867848) 2024-11-24T13:27:47,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741906_1084 (size=93) 2024-11-24T13:27:47,903 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:47,904 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:47,918 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905, exclude list is [], retry=0 2024-11-24T13:27:47,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:47,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:47,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:47,923 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 2024-11-24T13:27:47,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:47,991 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d4a9029308c597bb51bd23508d89c905, NAME => 'testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:47,994 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,994 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:47,994 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,994 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,996 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,997 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName a 2024-11-24T13:27:47,997 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,997 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,997 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,998 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName b 2024-11-24T13:27:47,998 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:47,998 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:47,999 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:47,999 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName c 2024-11-24T13:27:47,999 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:48,000 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:48,000 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,001 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,003 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,003 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,003 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:48,005 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,006 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d4a9029308c597bb51bd23508d89c905; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68960694, jitterRate=0.027594417333602905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:48,006 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:48,006 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d4a9029308c597bb51bd23508d89c905: Running coprocessor pre-open hook at 1732454867994Writing region info on filesystem at 1732454867994Initializing all the Stores at 1732454867995 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867995Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867995Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454867995Cleaning up temporary data from old regions at 1732454868003 (+8 ms)Running coprocessor post-open hooks at 1732454868006 (+3 ms)Region opened successfully at 1732454868006 2024-11-24T13:27:48,018 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d4a9029308c597bb51bd23508d89c905 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T13:27:48,019 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:48,493 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T13:27:48,758 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T13:27:48,758 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:48,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T13:27:48,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:48,760 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T13:27:48,760 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:48,761 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T13:27:48,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:48,763 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T13:27:48,763 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:49,020 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:49,386 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T13:27:50,021 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:51,021 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:52,022 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:53,023 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:54,023 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:55,024 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:56,024 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:57,025 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d4a9029308c597bb51bd23508d89c905/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:27:57,026 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d4a9029308c597bb51bd23508d89c905: 2024-11-24T13:27:57,026 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:57,040 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d4a9029308c597bb51bd23508d89c905: 2024-11-24T13:27:57,040 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T13:27:57,040 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d4a9029308c597bb51bd23508d89c905, disabling compactions & flushes 2024-11-24T13:27:57,040 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:57,040 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:57,040 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. after waiting 0 ms 2024-11-24T13:27:57,040 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:57,041 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:57,041 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. 2024-11-24T13:27:57,041 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d4a9029308c597bb51bd23508d89c905: Waiting for close lock at 1732454877040Running coprocessor pre-close hooks at 1732454877040Disabling compacts and flushes for region at 1732454877040Disabling writes for close at 1732454877040Writing region close event to WAL at 1732454877040Running coprocessor post-close hooks at 1732454877041 (+1 ms)Closed at 1732454877041 2024-11-24T13:27:57,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741908_1086 (size=3941) 2024-11-24T13:27:57,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741908_1086 (size=3941) 2024-11-24T13:27:57,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741908_1086 (size=3941) 2024-11-24T13:27:57,060 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905, size=3.8 K (3941bytes) 2024-11-24T13:27:57,060 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 2024-11-24T13:27:57,061 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 after 0ms 2024-11-24T13:27:57,063 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 took 3ms 2024-11-24T13:27:57,065 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 so closing down 2024-11-24T13:27:57,065 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:57,066 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732454867905.temp 2024-11-24T13:27:57,067 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000004-wal.1732454867905.temp 2024-11-24T13:27:57,068 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741909_1087 (size=2983) 2024-11-24T13:27:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741909_1087 (size=2983) 2024-11-24T13:27:57,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741909_1087 (size=2983) 2024-11-24T13:27:57,074 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000004-wal.1732454867905.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:57,075 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000004-wal.1732454867905.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000026 2024-11-24T13:27:57,075 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-24T13:27:57,075 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905, journal: Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905, size=3.8 K (3941bytes) at 1732454877060Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 so closing down at 1732454877065 (+5 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000004-wal.1732454867905.temp at 1732454877067 (+2 ms)3 split writer threads finished at 1732454877068 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000004-wal.1732454867905.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732454877074 (+6 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000004-wal.1732454867905.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000026 at 1732454877075 (+1 ms)Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732454877075 2024-11-24T13:27:57,077 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454867905 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454867905 2024-11-24T13:27:57,078 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000026 2024-11-24T13:27:57,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:57,080 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:57,093 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454877080, exclude list is [], retry=0 2024-11-24T13:27:57,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:57,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:57,096 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:57,098 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454877080 2024-11-24T13:27:57,098 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:57,098 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d4a9029308c597bb51bd23508d89c905, NAME => 'testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:57,099 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,099 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:57,099 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,099 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,101 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,102 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName a 2024-11-24T13:27:57,102 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,102 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,102 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,103 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName b 2024-11-24T13:27:57,103 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,104 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,104 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,104 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4a9029308c597bb51bd23508d89c905 columnFamilyName c 2024-11-24T13:27:57,104 DEBUG [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,105 INFO [StoreOpener-d4a9029308c597bb51bd23508d89c905-1 {}] regionserver.HStore(327): Store=d4a9029308c597bb51bd23508d89c905/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,105 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,107 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000026 2024-11-24T13:27:57,110 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000026 2024-11-24T13:27:57,110 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d4a9029308c597bb51bd23508d89c905 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T13:27:57,125 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/a/3aa5de625b6b4314993467a2f4372616 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732454877031/Put/seqid=0 2024-11-24T13:27:57,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741911_1089 (size=5523) 2024-11-24T13:27:57,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741911_1089 (size=5523) 2024-11-24T13:27:57,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741911_1089 (size=5523) 2024-11-24T13:27:57,133 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/a/3aa5de625b6b4314993467a2f4372616 2024-11-24T13:27:57,153 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/b/35ee8dbf5c1241b9bdab13604ad5944d is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732454877027/Put/seqid=0 2024-11-24T13:27:57,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741912_1090 (size=5524) 2024-11-24T13:27:57,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741912_1090 (size=5524) 2024-11-24T13:27:57,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741912_1090 (size=5524) 2024-11-24T13:27:57,160 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/b/35ee8dbf5c1241b9bdab13604ad5944d 2024-11-24T13:27:57,180 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/c/f83fd7b3b27d47d689ca04167ef1c96c is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732454877028/Put/seqid=0 2024-11-24T13:27:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741913_1091 (size=5457) 2024-11-24T13:27:57,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741913_1091 (size=5457) 2024-11-24T13:27:57,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741913_1091 (size=5457) 2024-11-24T13:27:57,186 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/c/f83fd7b3b27d47d689ca04167ef1c96c 2024-11-24T13:27:57,192 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/a/3aa5de625b6b4314993467a2f4372616 as hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/a/3aa5de625b6b4314993467a2f4372616 2024-11-24T13:27:57,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/a/3aa5de625b6b4314993467a2f4372616, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T13:27:57,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/b/35ee8dbf5c1241b9bdab13604ad5944d as hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/b/35ee8dbf5c1241b9bdab13604ad5944d 2024-11-24T13:27:57,202 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/b/35ee8dbf5c1241b9bdab13604ad5944d, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T13:27:57,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/.tmp/c/f83fd7b3b27d47d689ca04167ef1c96c as hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/c/f83fd7b3b27d47d689ca04167ef1c96c 2024-11-24T13:27:57,207 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/c/f83fd7b3b27d47d689ca04167ef1c96c, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T13:27:57,208 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d4a9029308c597bb51bd23508d89c905 in 98ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T13:27:57,209 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/0000000000000000026 2024-11-24T13:27:57,210 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,210 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,211 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:57,212 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,214 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsAfterAbortingFlush/d4a9029308c597bb51bd23508d89c905/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T13:27:57,215 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d4a9029308c597bb51bd23508d89c905; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73930721, jitterRate=0.10165359079837799}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:57,215 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4a9029308c597bb51bd23508d89c905 2024-11-24T13:27:57,216 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d4a9029308c597bb51bd23508d89c905: Running coprocessor pre-open hook at 1732454877099Writing region info on filesystem at 1732454877099Initializing all the Stores at 1732454877100 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877100Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877100Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877100Obtaining lock to block concurrent updates at 1732454877110 (+10 ms)Preparing flush snapshotting stores in d4a9029308c597bb51bd23508d89c905 at 1732454877110Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732454877110Flushing stores of testReplayEditsAfterAbortingFlush,,1732454867844.d4a9029308c597bb51bd23508d89c905. at 1732454877110Flushing d4a9029308c597bb51bd23508d89c905/a: creating writer at 1732454877110Flushing d4a9029308c597bb51bd23508d89c905/a: appending metadata at 1732454877125 (+15 ms)Flushing d4a9029308c597bb51bd23508d89c905/a: closing flushed file at 1732454877125Flushing d4a9029308c597bb51bd23508d89c905/b: creating writer at 1732454877138 (+13 ms)Flushing d4a9029308c597bb51bd23508d89c905/b: appending metadata at 1732454877152 (+14 ms)Flushing d4a9029308c597bb51bd23508d89c905/b: closing flushed file at 1732454877152Flushing d4a9029308c597bb51bd23508d89c905/c: creating writer at 1732454877165 (+13 ms)Flushing d4a9029308c597bb51bd23508d89c905/c: appending metadata at 1732454877179 (+14 ms)Flushing d4a9029308c597bb51bd23508d89c905/c: closing flushed file at 1732454877179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37004896: reopening flushed file at 1732454877191 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c6893da: reopening flushed file at 1732454877197 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41b16829: reopening flushed file at 1732454877202 (+5 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d4a9029308c597bb51bd23508d89c905 in 98ms, sequenceid=26, compaction requested=false; wal=null at 1732454877208 (+6 ms)Cleaning up temporary data from old regions at 1732454877210 (+2 ms)Running coprocessor post-open hooks at 1732454877215 (+5 ms)Region opened successfully at 1732454877216 (+1 ms) 2024-11-24T13:27:57,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 419), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=7081 (was 7078) - AvailableMemoryMB LEAK? - 2024-11-24T13:27:57,241 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T13:27:57,254 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=7079 2024-11-24T13:27:57,254 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T13:27:57,269 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:57,271 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:57,271 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:57,274 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-87507277, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-87507277, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:57,287 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-87507277/hregion-87507277.1732454877274, exclude list is [], retry=0 2024-11-24T13:27:57,290 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:57,290 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:57,291 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:57,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-87507277/hregion-87507277.1732454877274 2024-11-24T13:27:57,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:57,293 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ec59b375fdb08fbf840323920386774c, NAME => 'testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741915_1093 (size=61) 2024-11-24T13:27:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741915_1093 (size=61) 2024-11-24T13:27:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741915_1093 (size=61) 2024-11-24T13:27:57,306 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:57,308 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,310 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec59b375fdb08fbf840323920386774c columnFamilyName a 2024-11-24T13:27:57,310 DEBUG [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,311 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(327): Store=ec59b375fdb08fbf840323920386774c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,311 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,312 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,312 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,314 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,316 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:57,316 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec59b375fdb08fbf840323920386774c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61677649, jitterRate=-0.0809314101934433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec59b375fdb08fbf840323920386774c: Writing region info on filesystem at 1732454877307Initializing all the Stores at 1732454877307Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877307Cleaning up temporary data from old regions at 1732454877312 (+5 ms)Region opened successfully at 1732454877316 (+4 ms) 2024-11-24T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec59b375fdb08fbf840323920386774c, disabling compactions & flushes 2024-11-24T13:27:57,317 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. after waiting 0 ms 2024-11-24T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,317 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec59b375fdb08fbf840323920386774c: Waiting for close lock at 1732454877317Disabling compacts and flushes for region at 1732454877317Disabling writes for close at 1732454877317Writing region close event to WAL at 1732454877317Closed at 1732454877317 2024-11-24T13:27:57,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741914_1092 (size=93) 2024-11-24T13:27:57,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741914_1092 (size=93) 2024-11-24T13:27:57,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741914_1092 (size=93) 2024-11-24T13:27:57,322 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:57,322 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-87507277:(num 1732454877274) 2024-11-24T13:27:57,322 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:57,324 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:57,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325, exclude list is [], retry=0 2024-11-24T13:27:57,347 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:57,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:57,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:57,349 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 2024-11-24T13:27:57,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:57,350 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ec59b375fdb08fbf840323920386774c, NAME => 'testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:57,350 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:57,350 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,350 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,351 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,352 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec59b375fdb08fbf840323920386774c columnFamilyName a 2024-11-24T13:27:57,352 DEBUG [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,353 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(327): Store=ec59b375fdb08fbf840323920386774c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,353 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,354 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,355 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,357 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,357 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec59b375fdb08fbf840323920386774c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73269310, jitterRate=0.09179779887199402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:57,358 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec59b375fdb08fbf840323920386774c: Writing region info on filesystem at 1732454877350Initializing all the Stores at 1732454877351 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877351Cleaning up temporary data from old regions at 1732454877355 (+4 ms)Region opened successfully at 1732454877358 (+3 ms) 2024-11-24T13:27:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec59b375fdb08fbf840323920386774c, disabling compactions & flushes 2024-11-24T13:27:57,366 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. after waiting 0 ms 2024-11-24T13:27:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,366 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,366 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. 2024-11-24T13:27:57,367 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec59b375fdb08fbf840323920386774c: Waiting for close lock at 1732454877366Disabling compacts and flushes for region at 1732454877366Disabling writes for close at 1732454877366Writing region close event to WAL at 1732454877366Closed at 1732454877366 2024-11-24T13:27:57,370 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 not finished, retry = 0 2024-11-24T13:27:57,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741916_1094 (size=1623) 2024-11-24T13:27:57,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741916_1094 (size=1623) 2024-11-24T13:27:57,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741916_1094 (size=1623) 2024-11-24T13:27:57,473 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T13:27:57,486 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325, size=1.6 K (1623bytes) 2024-11-24T13:27:57,487 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 2024-11-24T13:27:57,487 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 after 0ms 2024-11-24T13:27:57,489 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 took 3ms 2024-11-24T13:27:57,491 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 so closing down 2024-11-24T13:27:57,491 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:27:57,492 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732454877325.temp 2024-11-24T13:27:57,493 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000003-wal.1732454877325.temp 2024-11-24T13:27:57,494 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:27:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741917_1095 (size=1623) 2024-11-24T13:27:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741917_1095 (size=1623) 2024-11-24T13:27:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741917_1095 (size=1623) 2024-11-24T13:27:57,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000003-wal.1732454877325.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T13:27:57,503 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000003-wal.1732454877325.temp to hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000012 2024-11-24T13:27:57,504 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-24T13:27:57,504 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325, journal: Splitting hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325, size=1.6 K (1623bytes) at 1732454877487Finishing writing output for hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 so closing down at 1732454877491 (+4 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000003-wal.1732454877325.temp at 1732454877493 (+2 ms)3 split writer threads finished at 1732454877494 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000003-wal.1732454877325.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732454877502 (+8 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000003-wal.1732454877325.temp to hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000012 at 1732454877503 (+1 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732454877504 (+1 ms) 2024-11-24T13:27:57,505 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877325 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454877325 2024-11-24T13:27:57,506 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000012 2024-11-24T13:27:57,813 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:57,816 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:57,829 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877816, exclude list is [], retry=0 2024-11-24T13:27:57,832 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:57,833 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:57,833 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:57,834 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877816 2024-11-24T13:27:57,835 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:27:57,835 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ec59b375fdb08fbf840323920386774c, NAME => 'testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:57,835 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:57,835 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,835 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,838 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,840 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec59b375fdb08fbf840323920386774c columnFamilyName a 2024-11-24T13:27:57,840 DEBUG [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,841 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(327): Store=ec59b375fdb08fbf840323920386774c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,841 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,842 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,843 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000012 2024-11-24T13:27:57,846 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000012 2024-11-24T13:27:57,846 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec59b375fdb08fbf840323920386774c 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T13:27:57,865 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/.tmp/a/ca450920411a445eafff44a987e0aafb is 79, key is testDatalossWhenInputError/a:x0/1732454877358/Put/seqid=0 2024-11-24T13:27:57,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741919_1097 (size=5808) 2024-11-24T13:27:57,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741919_1097 (size=5808) 2024-11-24T13:27:57,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741919_1097 (size=5808) 2024-11-24T13:27:57,872 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/.tmp/a/ca450920411a445eafff44a987e0aafb 2024-11-24T13:27:57,881 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/.tmp/a/ca450920411a445eafff44a987e0aafb as hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/a/ca450920411a445eafff44a987e0aafb 2024-11-24T13:27:57,890 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/a/ca450920411a445eafff44a987e0aafb, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T13:27:57,890 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ec59b375fdb08fbf840323920386774c in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T13:27:57,891 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/0000000000000000012 2024-11-24T13:27:57,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,892 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,894 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,896 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T13:27:57,897 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec59b375fdb08fbf840323920386774c; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63628663, jitterRate=-0.05185903608798981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:57,898 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec59b375fdb08fbf840323920386774c: Writing region info on filesystem at 1732454877835Initializing all the Stores at 1732454877838 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877838Obtaining lock to block concurrent updates at 1732454877846 (+8 ms)Preparing flush snapshotting stores in ec59b375fdb08fbf840323920386774c at 1732454877846Finished memstore snapshotting testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732454877847 (+1 ms)Flushing stores of testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c. at 1732454877847Flushing ec59b375fdb08fbf840323920386774c/a: creating writer at 1732454877847Flushing ec59b375fdb08fbf840323920386774c/a: appending metadata at 1732454877864 (+17 ms)Flushing ec59b375fdb08fbf840323920386774c/a: closing flushed file at 1732454877864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60f05d6f: reopening flushed file at 1732454877880 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ec59b375fdb08fbf840323920386774c in 44ms, sequenceid=12, compaction requested=false; wal=null at 1732454877890 (+10 ms)Cleaning up temporary data from old regions at 1732454877892 (+2 ms)Region opened successfully at 1732454877898 (+6 ms) 2024-11-24T13:27:57,901 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ec59b375fdb08fbf840323920386774c, NAME => 'testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:57,901 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732454877270.ec59b375fdb08fbf840323920386774c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:57,901 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,901 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,902 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,903 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec59b375fdb08fbf840323920386774c columnFamilyName a 2024-11-24T13:27:57,903 DEBUG [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:57,908 DEBUG [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/a/ca450920411a445eafff44a987e0aafb 2024-11-24T13:27:57,908 INFO [StoreOpener-ec59b375fdb08fbf840323920386774c-1 {}] regionserver.HStore(327): Store=ec59b375fdb08fbf840323920386774c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:57,908 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,908 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,910 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,910 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,911 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec59b375fdb08fbf840323920386774c 2024-11-24T13:27:57,913 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testDatalossWhenInputError/ec59b375fdb08fbf840323920386774c/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T13:27:57,914 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec59b375fdb08fbf840323920386774c; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73225767, jitterRate=0.09114895761013031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:27:57,914 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec59b375fdb08fbf840323920386774c: Writing region info on filesystem at 1732454877901Initializing all the Stores at 1732454877902 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454877902Cleaning up temporary data from old regions at 1732454877910 (+8 ms)Region opened successfully at 1732454877914 (+4 ms) 2024-11-24T13:27:57,931 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:34976 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:35054 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:55936 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:55962 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:45942 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:46002 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1725618806-172.17.0.2-1732454845323:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 268) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7072 (was 7079) 2024-11-24T13:27:57,932 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T13:27:57,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=7072 2024-11-24T13:27:57,945 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T13:27:57,960 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:57,962 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:27:57,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:27:57,964 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72992103, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-72992103, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:57,977 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72992103/hregion-72992103.1732454877965, exclude list is [], retry=0 2024-11-24T13:27:57,980 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:57,980 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:57,980 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:57,985 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72992103/hregion-72992103.1732454877965 2024-11-24T13:27:57,986 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:57,986 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c0c3cf135532a378317c97ba3733c409, NAME => 'testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:27:57,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741921_1099 (size=63) 2024-11-24T13:27:57,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741921_1099 (size=63) 2024-11-24T13:27:57,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741921_1099 (size=63) 2024-11-24T13:27:58,001 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:58,003 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,004 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName a 2024-11-24T13:27:58,004 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:58,005 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:58,005 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,006 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName b 2024-11-24T13:27:58,006 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:58,007 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:58,007 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,008 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName c 2024-11-24T13:27:58,008 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:58,009 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:58,009 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,010 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,011 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,011 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,011 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:58,012 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,015 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:27:58,015 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c0c3cf135532a378317c97ba3733c409; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71157652, jitterRate=0.060331642627716064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:58,015 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c0c3cf135532a378317c97ba3733c409: Writing region info on filesystem at 1732454878001Initializing all the Stores at 1732454878002 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454878002Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454878002Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454878002Cleaning up temporary data from old regions at 1732454878011 (+9 ms)Region opened successfully at 1732454878015 (+4 ms) 2024-11-24T13:27:58,016 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c0c3cf135532a378317c97ba3733c409, disabling compactions & flushes 2024-11-24T13:27:58,016 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:27:58,016 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:27:58,016 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. after waiting 0 ms 2024-11-24T13:27:58,016 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:27:58,016 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:27:58,016 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c0c3cf135532a378317c97ba3733c409: Waiting for close lock at 1732454878016Disabling compacts and flushes for region at 1732454878016Disabling writes for close at 1732454878016Writing region close event to WAL at 1732454878016Closed at 1732454878016 2024-11-24T13:27:58,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741920_1098 (size=93) 2024-11-24T13:27:58,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741920_1098 (size=93) 2024-11-24T13:27:58,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741920_1098 (size=93) 2024-11-24T13:27:58,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:27:58,023 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72992103:(num 1732454877965) 2024-11-24T13:27:58,023 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:27:58,024 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:27:58,038 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025, exclude list is [], retry=0 2024-11-24T13:27:58,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:27:58,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:27:58,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:27:58,042 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 2024-11-24T13:27:58,042 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:27:58,042 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c0c3cf135532a378317c97ba3733c409, NAME => 'testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:27:58,043 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:27:58,043 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,043 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,044 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,045 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName a 2024-11-24T13:27:58,045 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:58,046 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:58,046 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,047 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName b 2024-11-24T13:27:58,047 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:58,047 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:58,048 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,048 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName c 2024-11-24T13:27:58,049 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:27:58,049 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:27:58,049 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,050 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,051 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,052 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,053 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,053 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:27:58,055 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:27:58,055 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c0c3cf135532a378317c97ba3733c409; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68827321, jitterRate=0.025607004761695862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:27:58,056 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c0c3cf135532a378317c97ba3733c409: Writing region info on filesystem at 1732454878043Initializing all the Stores at 1732454878044 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454878044Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454878044Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454878044Cleaning up temporary data from old regions at 1732454878053 (+9 ms)Region opened successfully at 1732454878056 (+3 ms) 2024-11-24T13:27:58,059 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732454878059/Put/seqid=0 2024-11-24T13:27:58,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741923_1101 (size=4875) 2024-11-24T13:27:58,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741923_1101 (size=4875) 2024-11-24T13:27:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741923_1101 (size=4875) 2024-11-24T13:27:58,077 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732454878077/Put/seqid=0 2024-11-24T13:27:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741924_1102 (size=4875) 2024-11-24T13:27:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741924_1102 (size=4875) 2024-11-24T13:27:58,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741924_1102 (size=4875) 2024-11-24T13:27:58,086 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732454878086/Put/seqid=0 2024-11-24T13:27:58,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741925_1103 (size=4875) 2024-11-24T13:27:58,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741925_1103 (size=4875) 2024-11-24T13:27:58,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741925_1103 (size=4875) 2024-11-24T13:27:58,095 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c0c3cf135532a378317c97ba3733c409/a 2024-11-24T13:27:58,099 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T13:27:58,099 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T13:27:58,100 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c0c3cf135532a378317c97ba3733c409/a 2024-11-24T13:27:58,103 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T13:27:58,103 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T13:27:58,103 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c0c3cf135532a378317c97ba3733c409/a 2024-11-24T13:27:58,107 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T13:27:58,107 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T13:27:58,107 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c0c3cf135532a378317c97ba3733c409 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T13:27:58,130 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/.tmp/a/80a0df62363a4c48b54fad5d9eda1508 is 55, key is testCompactedBulkLoadedFiles/a:a/1732454878056/Put/seqid=0 2024-11-24T13:27:58,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741926_1104 (size=5107) 2024-11-24T13:27:58,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741926_1104 (size=5107) 2024-11-24T13:27:58,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741926_1104 (size=5107) 2024-11-24T13:27:58,139 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/.tmp/a/80a0df62363a4c48b54fad5d9eda1508 2024-11-24T13:27:58,144 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/.tmp/a/80a0df62363a4c48b54fad5d9eda1508 as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508 2024-11-24T13:27:58,148 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T13:27:58,149 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c0c3cf135532a378317c97ba3733c409 in 42ms, sequenceid=4, compaction requested=false 2024-11-24T13:27:58,149 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c0c3cf135532a378317c97ba3733c409: 2024-11-24T13:27:58,150 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ 2024-11-24T13:27:58,151 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ 2024-11-24T13:27:58,152 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ 2024-11-24T13:27:58,153 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile0 into c0c3cf135532a378317c97ba3733c409/a as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ - updating store file list. 2024-11-24T13:27:58,157 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3bfb50830398475ba1b96d14bf2d0333_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T13:27:58,157 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ into c0c3cf135532a378317c97ba3733c409/a 2024-11-24T13:27:58,157 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile0 into c0c3cf135532a378317c97ba3733c409/a (new location: hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_) 2024-11-24T13:27:58,158 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile1 into c0c3cf135532a378317c97ba3733c409/a as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ - updating store file list. 2024-11-24T13:27:58,161 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d5b2920f9f87417e9c914a6d269afa76_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T13:27:58,161 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ into c0c3cf135532a378317c97ba3733c409/a 2024-11-24T13:27:58,161 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile1 into c0c3cf135532a378317c97ba3733c409/a (new location: hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_) 2024-11-24T13:27:58,162 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile2 into c0c3cf135532a378317c97ba3733c409/a as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ - updating store file list. 2024-11-24T13:27:58,166 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T13:27:58,166 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ into c0c3cf135532a378317c97ba3733c409/a 2024-11-24T13:27:58,166 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35337/hbase/testCompactedBulkLoadedFiles/hfile2 into c0c3cf135532a378317c97ba3733c409/a (new location: hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_) 2024-11-24T13:27:58,173 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T13:27:58,173 DEBUG [Time-limited test {}] regionserver.HStore(1541): c0c3cf135532a378317c97ba3733c409/a is initiating major compaction (all files) 2024-11-24T13:27:58,173 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c0c3cf135532a378317c97ba3733c409/a in testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:27:58,174 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_] into tmpdir=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/.tmp, totalSize=19.3 K 2024-11-24T13:27:58,174 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 80a0df62363a4c48b54fad5d9eda1508, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732454878056 2024-11-24T13:27:58,175 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3bfb50830398475ba1b96d14bf2d0333_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T13:27:58,175 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d5b2920f9f87417e9c914a6d269afa76_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T13:27:58,175 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T13:27:58,186 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/.tmp/a/6b5b7f60acdd49fea485a6123cfa76b2 is 55, key is testCompactedBulkLoadedFiles/a:a/1732454878056/Put/seqid=0 2024-11-24T13:27:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741927_1105 (size=6154) 2024-11-24T13:27:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741927_1105 (size=6154) 2024-11-24T13:27:58,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741927_1105 (size=6154) 2024-11-24T13:27:58,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/.tmp/a/6b5b7f60acdd49fea485a6123cfa76b2 as hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/6b5b7f60acdd49fea485a6123cfa76b2 2024-11-24T13:27:58,202 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c0c3cf135532a378317c97ba3733c409/a of c0c3cf135532a378317c97ba3733c409 into 6b5b7f60acdd49fea485a6123cfa76b2(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T13:27:58,202 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c0c3cf135532a378317c97ba3733c409: 2024-11-24T13:27:58,202 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T13:27:58,202 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T13:27:58,238 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025, size=0 (0bytes) 2024-11-24T13:27:58,238 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 might be still open, length is 0 2024-11-24T13:27:58,238 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 2024-11-24T13:27:58,238 WARN [IPC Server handler 1 on default port 35337 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-24T13:27:58,238 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 after 0ms 2024-11-24T13:27:58,758 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T13:27:58,758 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T13:27:58,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T13:27:58,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T13:28:00,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:46036 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:45659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46036 dst: /127.0.0.1:45659 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45659 remote=/127.0.0.1:46036]. Total timeout mills is 60000, 57925 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:00,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:35098 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:36111:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35098 dst: /127.0.0.1:36111 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:00,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:56000 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:46321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56000 dst: /127.0.0.1:46321 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741922_1106 (size=1659) 2024-11-24T13:28:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741922_1106 (size=1659) 2024-11-24T13:28:00,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741922_1106 (size=1659) 2024-11-24T13:28:02,239 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 after 4001ms 2024-11-24T13:28:02,242 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 took 4005ms 2024-11-24T13:28:02,243 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025; continuing. 2024-11-24T13:28:02,244 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 so closing down 2024-11-24T13:28:02,244 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:28:02,245 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732454878025.temp 2024-11-24T13:28:02,246 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000003-wal.1732454878025.temp 2024-11-24T13:28:02,246 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:28:02,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741928_1107 (size=698) 2024-11-24T13:28:02,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741928_1107 (size=698) 2024-11-24T13:28:02,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741928_1107 (size=698) 2024-11-24T13:28:02,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000003-wal.1732454878025.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T13:28:02,256 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000003-wal.1732454878025.temp to hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000008 2024-11-24T13:28:02,256 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T13:28:02,256 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025, journal: Splitting hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025, size=0 (0bytes) at 1732454878238Finishing writing output for hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 so closing down at 1732454882244 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000003-wal.1732454878025.temp at 1732454882246 (+2 ms)3 split writer threads finished at 1732454882246Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000003-wal.1732454878025.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732454882255 (+9 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000003-wal.1732454878025.temp to hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000008 at 1732454882256 (+1 ms)Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025, size=0, length=0, corrupted=false, cancelled=false at 1732454882256 2024-11-24T13:28:02,258 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454878025 2024-11-24T13:28:02,258 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000008 2024-11-24T13:28:02,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:28:02,260 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:28:02,274 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454882260, exclude list is [], retry=0 2024-11-24T13:28:02,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:28:02,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:28:02,277 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:28:02,278 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454882260 2024-11-24T13:28:02,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:28:02,279 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c0c3cf135532a378317c97ba3733c409, NAME => 'testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:28:02,279 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:28:02,279 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,279 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,280 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,281 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName a 2024-11-24T13:28:02,281 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,288 DEBUG [StoreFileOpener-c0c3cf135532a378317c97ba3733c409-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3bfb50830398475ba1b96d14bf2d0333_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T13:28:02,288 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ 2024-11-24T13:28:02,291 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/6b5b7f60acdd49fea485a6123cfa76b2 2024-11-24T13:28:02,295 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508 2024-11-24T13:28:02,298 DEBUG [StoreFileOpener-c0c3cf135532a378317c97ba3733c409-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T13:28:02,299 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ 2024-11-24T13:28:02,302 DEBUG [StoreFileOpener-c0c3cf135532a378317c97ba3733c409-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d5b2920f9f87417e9c914a6d269afa76_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T13:28:02,302 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ 2024-11-24T13:28:02,302 WARN [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@414e89aa 2024-11-24T13:28:02,303 WARN [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@414e89aa 2024-11-24T13:28:02,303 WARN [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@414e89aa 2024-11-24T13:28:02,303 WARN [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@414e89aa 2024-11-24T13:28:02,303 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_] to archive 2024-11-24T13:28:02,303 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T13:28:02,305 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ to hdfs://localhost:35337/hbase/archive/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/3bfb50830398475ba1b96d14bf2d0333_SeqId_4_ 2024-11-24T13:28:02,306 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508 to hdfs://localhost:35337/hbase/archive/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/80a0df62363a4c48b54fad5d9eda1508 2024-11-24T13:28:02,307 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ to hdfs://localhost:35337/hbase/archive/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_ 2024-11-24T13:28:02,308 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ to hdfs://localhost:35337/hbase/archive/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/a/d5b2920f9f87417e9c914a6d269afa76_SeqId_4_ 2024-11-24T13:28:02,308 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,308 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,309 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName b 2024-11-24T13:28:02,309 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,309 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,309 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,310 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c3cf135532a378317c97ba3733c409 columnFamilyName c 2024-11-24T13:28:02,310 DEBUG [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,310 INFO [StoreOpener-c0c3cf135532a378317c97ba3733c409-1 {}] regionserver.HStore(327): Store=c0c3cf135532a378317c97ba3733c409/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,313 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000008 2024-11-24T13:28:02,316 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c0c3cf135532a378317c97ba3733c409 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c0c3cf135532a378317c97ba3733c409" family_name: "a" compaction_input: "80a0df62363a4c48b54fad5d9eda1508" compaction_input: "3bfb50830398475ba1b96d14bf2d0333_SeqId_4_" compaction_input: "d5b2920f9f87417e9c914a6d269afa76_SeqId_4_" compaction_input: "9bf234510abf425a81bd14fda6e0b9eb_SeqId_4_" compaction_output: "6b5b7f60acdd49fea485a6123cfa76b2" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T13:28:02,316 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T13:28:02,316 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000008 2024-11-24T13:28:02,317 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/0000000000000000008 2024-11-24T13:28:02,318 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,318 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,318 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:28:02,320 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c0c3cf135532a378317c97ba3733c409 2024-11-24T13:28:02,322 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testCompactedBulkLoadedFiles/c0c3cf135532a378317c97ba3733c409/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T13:28:02,323 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c0c3cf135532a378317c97ba3733c409; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62129924, jitterRate=-0.07419198751449585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:28:02,323 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c0c3cf135532a378317c97ba3733c409: Writing region info on filesystem at 1732454882279Initializing all the Stores at 1732454882280 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882280Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882280Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882280Cleaning up temporary data from old regions at 1732454882318 (+38 ms)Region opened successfully at 1732454882323 (+5 ms) 2024-11-24T13:28:02,325 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c0c3cf135532a378317c97ba3733c409, disabling compactions & flushes 2024-11-24T13:28:02,325 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:28:02,325 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:28:02,325 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. after waiting 0 ms 2024-11-24T13:28:02,325 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:28:02,326 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732454877961.c0c3cf135532a378317c97ba3733c409. 2024-11-24T13:28:02,326 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c0c3cf135532a378317c97ba3733c409: Waiting for close lock at 1732454882325Disabling compacts and flushes for region at 1732454882325Disabling writes for close at 1732454882325Writing region close event to WAL at 1732454882326 (+1 ms)Closed at 1732454882326 2024-11-24T13:28:02,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741929_1108 (size=93) 2024-11-24T13:28:02,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741929_1108 (size=93) 2024-11-24T13:28:02,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741929_1108 (size=93) 2024-11-24T13:28:02,331 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:28:02,331 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732454882260) 2024-11-24T13:28:02,347 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=442 (was 429) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1763629497_22 at /127.0.0.1:48834 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1763629497_22 at /127.0.0.1:41034 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35337 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:35337 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1763629497_22 at /127.0.0.1:41462 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=7067 (was 7072) 2024-11-24T13:28:02,347 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T13:28:02,360 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=7065 2024-11-24T13:28:02,360 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T13:28:02,375 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:28:02,377 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:28:02,377 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:28:02,380 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16338477, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/hregion-16338477, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:28:02,393 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16338477/hregion-16338477.1732454882380, exclude list is [], retry=0 2024-11-24T13:28:02,395 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:28:02,395 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:28:02,395 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:28:02,398 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16338477/hregion-16338477.1732454882380 2024-11-24T13:28:02,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:28:02,398 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f7102b0e4409b7e17169561862041ae3, NAME => 'testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35337/hbase 2024-11-24T13:28:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741931_1110 (size=67) 2024-11-24T13:28:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741931_1110 (size=67) 2024-11-24T13:28:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741931_1110 (size=67) 2024-11-24T13:28:02,407 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:28:02,408 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,410 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName a 2024-11-24T13:28:02,410 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,410 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,410 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,412 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName b 2024-11-24T13:28:02,412 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,412 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,412 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,413 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName c 2024-11-24T13:28:02,414 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,414 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,414 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,415 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,415 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,416 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,416 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,417 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:28:02,418 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,421 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:28:02,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f7102b0e4409b7e17169561862041ae3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70388135, jitterRate=0.04886494576931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:28:02,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f7102b0e4409b7e17169561862041ae3: Writing region info on filesystem at 1732454882407Initializing all the Stores at 1732454882408 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882408Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882408Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882408Cleaning up temporary data from old regions at 1732454882416 (+8 ms)Region opened successfully at 1732454882421 (+5 ms) 2024-11-24T13:28:02,421 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f7102b0e4409b7e17169561862041ae3, disabling compactions & flushes 2024-11-24T13:28:02,421 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,421 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,421 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. after waiting 0 ms 2024-11-24T13:28:02,421 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,422 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,422 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f7102b0e4409b7e17169561862041ae3: Waiting for close lock at 1732454882421Disabling compacts and flushes for region at 1732454882421Disabling writes for close at 1732454882421Writing region close event to WAL at 1732454882422 (+1 ms)Closed at 1732454882422 2024-11-24T13:28:02,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741930_1109 (size=93) 2024-11-24T13:28:02,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741930_1109 (size=93) 2024-11-24T13:28:02,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741930_1109 (size=93) 2024-11-24T13:28:02,427 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:28:02,427 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16338477:(num 1732454882380) 2024-11-24T13:28:02,427 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:28:02,429 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:28:02,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429, exclude list is [], retry=0 2024-11-24T13:28:02,445 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:28:02,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:28:02,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:28:02,447 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 2024-11-24T13:28:02,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443)] 2024-11-24T13:28:02,448 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f7102b0e4409b7e17169561862041ae3, NAME => 'testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:28:02,448 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:28:02,448 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,448 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,449 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,450 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName a 2024-11-24T13:28:02,451 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,451 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,451 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,452 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName b 2024-11-24T13:28:02,452 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,453 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,453 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,453 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName c 2024-11-24T13:28:02,453 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,454 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,454 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,456 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,456 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,457 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,457 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:28:02,459 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,459 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f7102b0e4409b7e17169561862041ae3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75009380, jitterRate=0.11772686243057251}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:28:02,460 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f7102b0e4409b7e17169561862041ae3: Writing region info on filesystem at 1732454882448Initializing all the Stores at 1732454882449 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882449Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882449Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882449Cleaning up temporary data from old regions at 1732454882457 (+8 ms)Region opened successfully at 1732454882459 (+2 ms) 2024-11-24T13:28:02,468 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f7102b0e4409b7e17169561862041ae3 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T13:28:02,485 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/a/8e804d5b79bb4145990ac5369c33b5a2 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732454882460/Put/seqid=0 2024-11-24T13:28:02,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741933_1112 (size=5958) 2024-11-24T13:28:02,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741933_1112 (size=5958) 2024-11-24T13:28:02,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741933_1112 (size=5958) 2024-11-24T13:28:02,493 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/a/8e804d5b79bb4145990ac5369c33b5a2 2024-11-24T13:28:02,498 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/a/8e804d5b79bb4145990ac5369c33b5a2 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/a/8e804d5b79bb4145990ac5369c33b5a2 2024-11-24T13:28:02,503 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/a/8e804d5b79bb4145990ac5369c33b5a2, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T13:28:02,504 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f7102b0e4409b7e17169561862041ae3 in 36ms, sequenceid=13, compaction requested=false 2024-11-24T13:28:02,504 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f7102b0e4409b7e17169561862041ae3: 2024-11-24T13:28:02,521 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f7102b0e4409b7e17169561862041ae3, disabling compactions & flushes 2024-11-24T13:28:02,521 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,521 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,521 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. after waiting 0 ms 2024-11-24T13:28:02,521 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,522 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,522 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:02,522 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f7102b0e4409b7e17169561862041ae3: Waiting for close lock at 1732454882521Disabling compacts and flushes for region at 1732454882521Disabling writes for close at 1732454882521Writing region close event to WAL at 1732454882522 (+1 ms)Closed at 1732454882522 2024-11-24T13:28:02,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741932_1111 (size=5869) 2024-11-24T13:28:02,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741932_1111 (size=5869) 2024-11-24T13:28:02,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741932_1111 (size=5869) 2024-11-24T13:28:02,541 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429, size=5.7 K (5869bytes) 2024-11-24T13:28:02,541 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 2024-11-24T13:28:02,542 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 after 1ms 2024-11-24T13:28:02,544 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 took 3ms 2024-11-24T13:28:02,546 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 so closing down 2024-11-24T13:28:02,546 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:28:02,547 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732454882429.temp 2024-11-24T13:28:02,548 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000003-wal.1732454882429.temp 2024-11-24T13:28:02,548 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:28:02,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741934_1113 (size=5223) 2024-11-24T13:28:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741934_1113 (size=5223) 2024-11-24T13:28:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741934_1113 (size=5223) 2024-11-24T13:28:02,557 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000003-wal.1732454882429.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T13:28:02,559 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000003-wal.1732454882429.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000035 2024-11-24T13:28:02,559 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-24T13:28:02,559 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429, journal: Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429, size=5.7 K (5869bytes) at 1732454882541Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 so closing down at 1732454882546 (+5 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000003-wal.1732454882429.temp at 1732454882548 (+2 ms)3 split writer threads finished at 1732454882548Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000003-wal.1732454882429.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732454882557 (+9 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000003-wal.1732454882429.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000035 at 1732454882559 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732454882559 2024-11-24T13:28:02,561 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882429 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454882429 2024-11-24T13:28:02,561 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000035 2024-11-24T13:28:02,562 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:28:02,563 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:28:02,566 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T13:28:02,580 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563, exclude list is [], retry=0 2024-11-24T13:28:02,582 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:28:02,582 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:28:02,583 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:28:02,584 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 2024-11-24T13:28:02,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655),(127.0.0.1/127.0.0.1:34299:34299)] 2024-11-24T13:28:02,585 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f7102b0e4409b7e17169561862041ae3, NAME => 'testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:28:02,585 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:28:02,585 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,585 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,586 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,587 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName a 2024-11-24T13:28:02,587 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,592 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/a/8e804d5b79bb4145990ac5369c33b5a2 2024-11-24T13:28:02,592 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,593 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,593 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName b 2024-11-24T13:28:02,593 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,594 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,594 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,594 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName c 2024-11-24T13:28:02,595 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:02,595 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:02,595 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,596 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,597 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000035 2024-11-24T13:28:02,600 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000035 2024-11-24T13:28:02,600 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f7102b0e4409b7e17169561862041ae3 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T13:28:02,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/b/46fc550d005a459cbfce502d71bf5e02 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732454882504/Put/seqid=0 2024-11-24T13:28:02,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741936_1115 (size=5958) 2024-11-24T13:28:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741936_1115 (size=5958) 2024-11-24T13:28:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741936_1115 (size=5958) 2024-11-24T13:28:02,623 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/b/46fc550d005a459cbfce502d71bf5e02 2024-11-24T13:28:02,642 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/c/5d2d754ec32244d8ac98ecd0c446f94d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732454882511/Put/seqid=0 2024-11-24T13:28:02,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741937_1116 (size=5958) 2024-11-24T13:28:02,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741937_1116 (size=5958) 2024-11-24T13:28:02,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741937_1116 (size=5958) 2024-11-24T13:28:02,651 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/c/5d2d754ec32244d8ac98ecd0c446f94d 2024-11-24T13:28:02,657 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/b/46fc550d005a459cbfce502d71bf5e02 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/b/46fc550d005a459cbfce502d71bf5e02 2024-11-24T13:28:02,661 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/b/46fc550d005a459cbfce502d71bf5e02, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T13:28:02,663 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/c/5d2d754ec32244d8ac98ecd0c446f94d as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/c/5d2d754ec32244d8ac98ecd0c446f94d 2024-11-24T13:28:02,667 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/c/5d2d754ec32244d8ac98ecd0c446f94d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T13:28:02,667 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f7102b0e4409b7e17169561862041ae3 in 67ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T13:28:02,668 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000035 2024-11-24T13:28:02,669 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,669 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,670 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:28:02,671 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:02,673 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T13:28:02,674 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f7102b0e4409b7e17169561862041ae3; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74124019, jitterRate=0.10453395545482635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:28:02,674 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f7102b0e4409b7e17169561862041ae3: Writing region info on filesystem at 1732454882585Initializing all the Stores at 1732454882586 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882586Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882586Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454882586Obtaining lock to block concurrent updates at 1732454882600 (+14 ms)Preparing flush snapshotting stores in f7102b0e4409b7e17169561862041ae3 at 1732454882600Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732454882600Flushing stores of testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. at 1732454882600Flushing f7102b0e4409b7e17169561862041ae3/b: creating writer at 1732454882600Flushing f7102b0e4409b7e17169561862041ae3/b: appending metadata at 1732454882614 (+14 ms)Flushing f7102b0e4409b7e17169561862041ae3/b: closing flushed file at 1732454882614Flushing f7102b0e4409b7e17169561862041ae3/c: creating writer at 1732454882628 (+14 ms)Flushing f7102b0e4409b7e17169561862041ae3/c: appending metadata at 1732454882641 (+13 ms)Flushing f7102b0e4409b7e17169561862041ae3/c: closing flushed file at 1732454882641Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25dc9bdd: reopening flushed file at 1732454882656 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20338579: reopening flushed file at 1732454882662 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f7102b0e4409b7e17169561862041ae3 in 67ms, sequenceid=35, compaction requested=false; wal=null at 1732454882667 (+5 ms)Cleaning up temporary data from old regions at 1732454882669 (+2 ms)Region opened successfully at 1732454882674 (+5 ms) 2024-11-24T13:28:02,758 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563, size=0 (0bytes) 2024-11-24T13:28:02,759 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 might be still open, length is 0 2024-11-24T13:28:02,759 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 2024-11-24T13:28:02,759 WARN [IPC Server handler 1 on default port 35337 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-24T13:28:02,759 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 after 0ms 2024-11-24T13:28:03,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:41090 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:45659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41090 dst: /127.0.0.1:45659 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45659 remote=/127.0.0.1:41090]. Total timeout mills is 60000, 59426 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:03,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:41522 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:36111:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41522 dst: /127.0.0.1:36111 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:03,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1106905432_22 at /127.0.0.1:48874 [Receiving block BP-1725618806-172.17.0.2-1732454845323:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:46321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48874 dst: /127.0.0.1:46321 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:03,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741935_1117 (size=5215) 2024-11-24T13:28:03,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741935_1117 (size=5215) 2024-11-24T13:28:03,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741935_1117 (size=5215) 2024-11-24T13:28:06,760 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 after 4001ms 2024-11-24T13:28:06,763 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 took 4005ms 2024-11-24T13:28:06,765 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563; continuing. 2024-11-24T13:28:06,765 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 so closing down 2024-11-24T13:28:06,765 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T13:28:06,766 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732454882563.temp 2024-11-24T13:28:06,767 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000037-wal.1732454882563.temp 2024-11-24T13:28:06,767 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T13:28:06,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741938_1118 (size=5223) 2024-11-24T13:28:06,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741938_1118 (size=5223) 2024-11-24T13:28:06,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741938_1118 (size=5223) 2024-11-24T13:28:06,774 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000037-wal.1732454882563.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T13:28:06,775 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000037-wal.1732454882563.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000066 2024-11-24T13:28:06,775 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T13:28:06,775 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563, journal: Splitting hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563, size=0 (0bytes) at 1732454882758Finishing writing output for hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 so closing down at 1732454886765 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000037-wal.1732454882563.temp at 1732454886767 (+2 ms)3 split writer threads finished at 1732454886767Closed recovered edits writer path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000037-wal.1732454882563.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732454886774 (+7 ms)Rename recovered edits hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000037-wal.1732454882563.temp to hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000066 at 1732454886775 (+1 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563, size=0, length=0, corrupted=false, cancelled=false at 1732454886775 2024-11-24T13:28:06,777 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 to hdfs://localhost:35337/hbase/oldWALs/wal.1732454882563 2024-11-24T13:28:06,777 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000066 2024-11-24T13:28:06,777 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T13:28:06,779 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35337/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374, archiveDir=hdfs://localhost:35337/hbase/oldWALs, maxLogs=32 2024-11-24T13:28:06,792 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454886779, exclude list is [], retry=0 2024-11-24T13:28:06,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46321,DS-36c3a7b6-0bbf-40e3-915b-d778e94d8abe,DISK] 2024-11-24T13:28:06,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45659,DS-cf01788b-2b1a-4b62-b0e5-9187cf456c4a,DISK] 2024-11-24T13:28:06,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36111,DS-b5fe2a2b-d189-4a0a-b6bc-74050c4f3d12,DISK] 2024-11-24T13:28:06,796 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454886779 2024-11-24T13:28:06,797 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34299:34299),(127.0.0.1/127.0.0.1:38443:38443),(127.0.0.1/127.0.0.1:40655:40655)] 2024-11-24T13:28:06,797 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:28:06,798 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,799 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName a 2024-11-24T13:28:06,799 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:06,804 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/a/8e804d5b79bb4145990ac5369c33b5a2 2024-11-24T13:28:06,804 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:06,804 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,804 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName b 2024-11-24T13:28:06,805 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:06,808 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/b/46fc550d005a459cbfce502d71bf5e02 2024-11-24T13:28:06,809 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:06,809 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,809 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7102b0e4409b7e17169561862041ae3 columnFamilyName c 2024-11-24T13:28:06,809 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:28:06,814 DEBUG [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/c/5d2d754ec32244d8ac98ecd0c446f94d 2024-11-24T13:28:06,814 INFO [StoreOpener-f7102b0e4409b7e17169561862041ae3-1 {}] regionserver.HStore(327): Store=f7102b0e4409b7e17169561862041ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:28:06,814 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,815 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,816 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,817 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000066 2024-11-24T13:28:06,823 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000066 2024-11-24T13:28:06,823 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f7102b0e4409b7e17169561862041ae3 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T13:28:06,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/a/96ece0b76c4f4ea58e62e7387addcc9a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732454882680/Put/seqid=0 2024-11-24T13:28:06,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741940_1120 (size=5958) 2024-11-24T13:28:06,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741940_1120 (size=5958) 2024-11-24T13:28:06,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741940_1120 (size=5958) 2024-11-24T13:28:06,845 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/a/96ece0b76c4f4ea58e62e7387addcc9a 2024-11-24T13:28:06,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/b/305a9ee230db4cff92cc9e0bf4e998aa is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732454882687/Put/seqid=0 2024-11-24T13:28:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741941_1121 (size=5958) 2024-11-24T13:28:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741941_1121 (size=5958) 2024-11-24T13:28:06,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741941_1121 (size=5958) 2024-11-24T13:28:06,870 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/b/305a9ee230db4cff92cc9e0bf4e998aa 2024-11-24T13:28:06,888 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/c/b095b6fd3f344f6593dec445b26b3f52 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732454882694/Put/seqid=0 2024-11-24T13:28:06,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741942_1122 (size=5958) 2024-11-24T13:28:06,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741942_1122 (size=5958) 2024-11-24T13:28:06,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741942_1122 (size=5958) 2024-11-24T13:28:06,894 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/c/b095b6fd3f344f6593dec445b26b3f52 2024-11-24T13:28:06,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/a/96ece0b76c4f4ea58e62e7387addcc9a as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/a/96ece0b76c4f4ea58e62e7387addcc9a 2024-11-24T13:28:06,903 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/a/96ece0b76c4f4ea58e62e7387addcc9a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T13:28:06,904 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/b/305a9ee230db4cff92cc9e0bf4e998aa as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/b/305a9ee230db4cff92cc9e0bf4e998aa 2024-11-24T13:28:06,907 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/b/305a9ee230db4cff92cc9e0bf4e998aa, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T13:28:06,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/.tmp/c/b095b6fd3f344f6593dec445b26b3f52 as hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/c/b095b6fd3f344f6593dec445b26b3f52 2024-11-24T13:28:06,912 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/c/b095b6fd3f344f6593dec445b26b3f52, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T13:28:06,913 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f7102b0e4409b7e17169561862041ae3 in 90ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T13:28:06,913 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/0000000000000000066 2024-11-24T13:28:06,914 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,914 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,915 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T13:28:06,916 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f7102b0e4409b7e17169561862041ae3 2024-11-24T13:28:06,917 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/hbase/data/default/testReplayEditsWrittenViaHRegion/f7102b0e4409b7e17169561862041ae3/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T13:28:06,918 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f7102b0e4409b7e17169561862041ae3; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64804454, jitterRate=-0.03433838486671448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T13:28:06,918 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f7102b0e4409b7e17169561862041ae3: Writing region info on filesystem at 1732454886797Initializing all the Stores at 1732454886798 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454886798Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454886798Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732454886798Obtaining lock to block concurrent updates at 1732454886823 (+25 ms)Preparing flush snapshotting stores in f7102b0e4409b7e17169561862041ae3 at 1732454886823Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732454886823Flushing stores of testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. at 1732454886823Flushing f7102b0e4409b7e17169561862041ae3/a: creating writer at 1732454886823Flushing f7102b0e4409b7e17169561862041ae3/a: appending metadata at 1732454886837 (+14 ms)Flushing f7102b0e4409b7e17169561862041ae3/a: closing flushed file at 1732454886837Flushing f7102b0e4409b7e17169561862041ae3/b: creating writer at 1732454886849 (+12 ms)Flushing f7102b0e4409b7e17169561862041ae3/b: appending metadata at 1732454886863 (+14 ms)Flushing f7102b0e4409b7e17169561862041ae3/b: closing flushed file at 1732454886863Flushing f7102b0e4409b7e17169561862041ae3/c: creating writer at 1732454886874 (+11 ms)Flushing f7102b0e4409b7e17169561862041ae3/c: appending metadata at 1732454886888 (+14 ms)Flushing f7102b0e4409b7e17169561862041ae3/c: closing flushed file at 1732454886888Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d2e97ed: reopening flushed file at 1732454886898 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27c7a7c4: reopening flushed file at 1732454886903 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3502cbc6: reopening flushed file at 1732454886908 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f7102b0e4409b7e17169561862041ae3 in 90ms, sequenceid=66, compaction requested=false; wal=null at 1732454886913 (+5 ms)Cleaning up temporary data from old regions at 1732454886914 (+1 ms)Region opened successfully at 1732454886918 (+4 ms) 2024-11-24T13:28:06,930 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f7102b0e4409b7e17169561862041ae3, disabling compactions & flushes 2024-11-24T13:28:06,930 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:06,930 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:06,930 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. after waiting 0 ms 2024-11-24T13:28:06,930 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:06,931 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732454882376.f7102b0e4409b7e17169561862041ae3. 2024-11-24T13:28:06,931 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f7102b0e4409b7e17169561862041ae3: Waiting for close lock at 1732454886930Disabling compacts and flushes for region at 1732454886930Disabling writes for close at 1732454886930Writing region close event to WAL at 1732454886931 (+1 ms)Closed at 1732454886931 2024-11-24T13:28:06,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741939_1119 (size=93) 2024-11-24T13:28:06,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741939_1119 (size=93) 2024-11-24T13:28:06,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741939_1119 (size=93) 2024-11-24T13:28:06,936 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T13:28:06,936 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732454886779) 2024-11-24T13:28:06,950 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=449 (was 442) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35337 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2108437966_22 at /127.0.0.1:41128 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:35337 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2108437966_22 at /127.0.0.1:48916 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2108437966_22 at /127.0.0.1:41566 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=7050 (was 7065) 2024-11-24T13:28:06,950 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-11-24T13:28:06,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T13:28:06,950 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T13:28:06,950 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:28:06,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:06,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:06,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T13:28:06,951 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T13:28:06,951 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=814710804, stopped=false 2024-11-24T13:28:06,952 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b10924491aa5,38765,1732454848333 2024-11-24T13:28:06,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:28:06,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:28:06,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:28:06,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:28:06,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:28:06,954 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T13:28:06,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:28:06,954 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T13:28:06,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:28:06,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:06,954 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b10924491aa5,38813,1732454849031' ***** 2024-11-24T13:28:06,954 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T13:28:06,954 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b10924491aa5,45225,1732454849178' ***** 2024-11-24T13:28:06,954 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T13:28:06,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:28:06,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:28:06,955 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:28:06,955 INFO [RS:2;b10924491aa5:45225 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:28:06,955 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:28:06,955 INFO [RS:2;b10924491aa5:45225 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:28:06,955 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(3091): Received CLOSE for abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(959): stopping server b10924491aa5,38813,1732454849031 2024-11-24T13:28:06,955 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b10924491aa5:38813. 2024-11-24T13:28:06,955 DEBUG [RS:0;b10924491aa5:38813 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:28:06,955 DEBUG [RS:0;b10924491aa5:38813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:06,955 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:28:06,955 INFO [RS:0;b10924491aa5:38813 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:28:06,955 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(959): stopping server b10924491aa5,45225,1732454849178 2024-11-24T13:28:06,955 INFO [RS:2;b10924491aa5:45225 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:28:06,956 INFO [RS:2;b10924491aa5:45225 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b10924491aa5:45225. 2024-11-24T13:28:06,956 DEBUG [RS:2;b10924491aa5:45225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:28:06,956 DEBUG [RS:2;b10924491aa5:45225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:06,956 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T13:28:06,956 INFO [RS:0;b10924491aa5:38813 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:28:06,956 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1325): Online Regions={abd2f3ec3c78b40d57714a27dadd50db=testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db.} 2024-11-24T13:28:06,956 INFO [RS:0;b10924491aa5:38813 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:28:06,956 DEBUG [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1351): Waiting on abd2f3ec3c78b40d57714a27dadd50db 2024-11-24T13:28:06,956 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T13:28:06,956 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing abd2f3ec3c78b40d57714a27dadd50db, disabling compactions & flushes 2024-11-24T13:28:06,956 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:28:06,956 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:28:06,956 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. after waiting 0 ms 2024-11-24T13:28:06,956 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T13:28:06,956 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:28:06,956 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T13:28:06,956 DEBUG [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T13:28:06,957 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:28:06,957 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:28:06,957 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:28:06,957 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:28:06,957 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:28:06,957 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-24T13:28:06,960 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/default/testReplayEditsAfterRegionMovedWithMultiCF/abd2f3ec3c78b40d57714a27dadd50db/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T13:28:06,961 INFO [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:28:06,961 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for abd2f3ec3c78b40d57714a27dadd50db: Waiting for close lock at 1732454886956Running coprocessor pre-close hooks at 1732454886956Disabling compacts and flushes for region at 1732454886956Disabling writes for close at 1732454886956Writing region close event to WAL at 1732454886957 (+1 ms)Running coprocessor post-close hooks at 1732454886961 (+4 ms)Closed at 1732454886961 2024-11-24T13:28:06,961 DEBUG [RS_CLOSE_REGION-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db. 2024-11-24T13:28:06,962 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/info/098f18db77064f86bd136c6d1af4a8ed is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732454863459.abd2f3ec3c78b40d57714a27dadd50db./info:regioninfo/1732454867212/Put/seqid=0 2024-11-24T13:28:06,965 INFO [regionserver/b10924491aa5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:28:06,965 INFO [regionserver/b10924491aa5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:28:06,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741943_1123 (size=6778) 2024-11-24T13:28:06,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741943_1123 (size=6778) 2024-11-24T13:28:06,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741943_1123 (size=6778) 2024-11-24T13:28:06,968 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/info/098f18db77064f86bd136c6d1af4a8ed 2024-11-24T13:28:06,974 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/.tmp/info/098f18db77064f86bd136c6d1af4a8ed as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/info/098f18db77064f86bd136c6d1af4a8ed 2024-11-24T13:28:06,978 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/info/098f18db77064f86bd136c6d1af4a8ed, entries=8, sequenceid=23, filesize=6.6 K 2024-11-24T13:28:06,979 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=23, compaction requested=false 2024-11-24T13:28:06,983 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-24T13:28:06,984 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:28:06,984 INFO [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:28:06,984 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732454886956Running coprocessor pre-close hooks at 1732454886956Disabling compacts and flushes for region at 1732454886957 (+1 ms)Disabling writes for close at 1732454886957Obtaining lock to block concurrent updates at 1732454886957Preparing flush snapshotting stores in 1588230740 at 1732454886957Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732454886957Flushing stores of hbase:meta,,1.1588230740 at 1732454886958 (+1 ms)Flushing 1588230740/info: creating writer at 1732454886958Flushing 1588230740/info: appending metadata at 1732454886961 (+3 ms)Flushing 1588230740/info: closing flushed file at 1732454886961Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21fa158c: reopening flushed file at 1732454886973 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=23, compaction requested=false at 1732454886979 (+6 ms)Writing region close event to WAL at 1732454886980 (+1 ms)Running coprocessor post-close hooks at 1732454886984 (+4 ms)Closed at 1732454886984 2024-11-24T13:28:06,984 DEBUG [RS_CLOSE_META-regionserver/b10924491aa5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T13:28:07,156 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(976): stopping server b10924491aa5,45225,1732454849178; all regions closed. 2024-11-24T13:28:07,157 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(976): stopping server b10924491aa5,38813,1732454849031; all regions closed. 2024-11-24T13:28:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741833_1009 (size=928) 2024-11-24T13:28:07,160 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/WALs/b10924491aa5,38813,1732454849031/b10924491aa5%2C38813%2C1732454849031.meta.1732454866403.meta not finished, retry = 0 2024-11-24T13:28:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741893_1071 (size=2422) 2024-11-24T13:28:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741833_1009 (size=928) 2024-11-24T13:28:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741893_1071 (size=2422) 2024-11-24T13:28:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741833_1009 (size=928) 2024-11-24T13:28:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741893_1071 (size=2422) 2024-11-24T13:28:07,162 DEBUG [RS:2;b10924491aa5:45225 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs 2024-11-24T13:28:07,162 INFO [RS:2;b10924491aa5:45225 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b10924491aa5%2C45225%2C1732454849178:(num 1732454850477) 2024-11-24T13:28:07,162 DEBUG [RS:2;b10924491aa5:45225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:07,162 INFO [RS:2;b10924491aa5:45225 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:28:07,162 INFO [RS:2;b10924491aa5:45225 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:28:07,162 INFO [RS:2;b10924491aa5:45225 {}] hbase.ChoreService(370): Chore service for: regionserver/b10924491aa5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T13:28:07,163 INFO [RS:2;b10924491aa5:45225 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:28:07,163 INFO [RS:2;b10924491aa5:45225 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:28:07,163 INFO [RS:2;b10924491aa5:45225 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:28:07,163 INFO [regionserver/b10924491aa5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:28:07,163 INFO [RS:2;b10924491aa5:45225 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:28:07,163 INFO [RS:2;b10924491aa5:45225 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45225 2024-11-24T13:28:07,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b10924491aa5,45225,1732454849178 2024-11-24T13:28:07,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:28:07,165 INFO [RS:2;b10924491aa5:45225 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:28:07,168 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b10924491aa5,45225,1732454849178] 2024-11-24T13:28:07,169 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b10924491aa5,45225,1732454849178 already deleted, retry=false 2024-11-24T13:28:07,169 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b10924491aa5,45225,1732454849178 expired; onlineServers=1 2024-11-24T13:28:07,262 DEBUG [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs 2024-11-24T13:28:07,262 INFO [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b10924491aa5%2C38813%2C1732454849031.meta:.meta(num 1732454866403) 2024-11-24T13:28:07,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741834_1010 (size=2598) 2024-11-24T13:28:07,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741834_1010 (size=2598) 2024-11-24T13:28:07,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741834_1010 (size=2598) 2024-11-24T13:28:07,267 DEBUG [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/oldWALs 2024-11-24T13:28:07,267 INFO [RS:0;b10924491aa5:38813 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b10924491aa5%2C38813%2C1732454849031:(num 1732454850477) 2024-11-24T13:28:07,267 DEBUG [RS:0;b10924491aa5:38813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:28:07,267 INFO [RS:0;b10924491aa5:38813 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:28:07,267 INFO [RS:0;b10924491aa5:38813 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:28:07,267 INFO [RS:0;b10924491aa5:38813 {}] hbase.ChoreService(370): Chore service for: regionserver/b10924491aa5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T13:28:07,267 INFO [RS:0;b10924491aa5:38813 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:28:07,267 INFO [regionserver/b10924491aa5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:28:07,268 INFO [RS:0;b10924491aa5:38813 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38813 2024-11-24T13:28:07,268 INFO [RS:2;b10924491aa5:45225 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:28:07,268 INFO [RS:2;b10924491aa5:45225 {}] regionserver.HRegionServer(1031): Exiting; stopping=b10924491aa5,45225,1732454849178; zookeeper connection closed. 2024-11-24T13:28:07,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:28:07,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45225-0x1014a28febd0003, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:28:07,269 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@213754c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@213754c9 2024-11-24T13:28:07,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b10924491aa5,38813,1732454849031 2024-11-24T13:28:07,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:28:07,270 INFO [RS:0;b10924491aa5:38813 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:28:07,271 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b10924491aa5,38813,1732454849031] 2024-11-24T13:28:07,273 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b10924491aa5,38813,1732454849031 already deleted, retry=false 2024-11-24T13:28:07,273 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b10924491aa5,38813,1732454849031 expired; onlineServers=0 2024-11-24T13:28:07,273 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b10924491aa5,38765,1732454848333' ***** 2024-11-24T13:28:07,273 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T13:28:07,273 INFO [M:0;b10924491aa5:38765 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:28:07,273 INFO [M:0;b10924491aa5:38765 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:28:07,273 DEBUG [M:0;b10924491aa5:38765 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T13:28:07,273 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T13:28:07,273 DEBUG [M:0;b10924491aa5:38765 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T13:28:07,273 DEBUG [master/b10924491aa5:0:becomeActiveMaster-HFileCleaner.large.0-1732454850142 {}] cleaner.HFileCleaner(306): Exit Thread[master/b10924491aa5:0:becomeActiveMaster-HFileCleaner.large.0-1732454850142,5,FailOnTimeoutGroup] 2024-11-24T13:28:07,273 DEBUG [master/b10924491aa5:0:becomeActiveMaster-HFileCleaner.small.0-1732454850143 {}] cleaner.HFileCleaner(306): Exit Thread[master/b10924491aa5:0:becomeActiveMaster-HFileCleaner.small.0-1732454850143,5,FailOnTimeoutGroup] 2024-11-24T13:28:07,274 INFO [M:0;b10924491aa5:38765 {}] hbase.ChoreService(370): Chore service for: master/b10924491aa5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T13:28:07,274 INFO [M:0;b10924491aa5:38765 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:28:07,274 DEBUG [M:0;b10924491aa5:38765 {}] master.HMaster(1795): Stopping service threads 2024-11-24T13:28:07,274 INFO [M:0;b10924491aa5:38765 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T13:28:07,274 INFO [M:0;b10924491aa5:38765 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T13:28:07,275 INFO [M:0;b10924491aa5:38765 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T13:28:07,275 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T13:28:07,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T13:28:07,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:28:07,275 DEBUG [M:0;b10924491aa5:38765 {}] zookeeper.ZKUtil(347): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T13:28:07,275 WARN [M:0;b10924491aa5:38765 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T13:28:07,276 INFO [M:0;b10924491aa5:38765 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/.lastflushedseqids 2024-11-24T13:28:07,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741944_1124 (size=119) 2024-11-24T13:28:07,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741944_1124 (size=119) 2024-11-24T13:28:07,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741944_1124 (size=119) 2024-11-24T13:28:07,288 INFO [M:0;b10924491aa5:38765 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T13:28:07,289 INFO [M:0;b10924491aa5:38765 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T13:28:07,289 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T13:28:07,289 INFO [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:28:07,289 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:28:07,289 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T13:28:07,289 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:28:07,289 INFO [M:0;b10924491aa5:38765 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-24T13:28:07,305 DEBUG [M:0;b10924491aa5:38765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2497726a6e4494bb4e1f9b86c2bf7cf is 82, key is hbase:meta,,1/info:regioninfo/1732454866560/Put/seqid=0 2024-11-24T13:28:07,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741945_1125 (size=6063) 2024-11-24T13:28:07,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741945_1125 (size=6063) 2024-11-24T13:28:07,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741945_1125 (size=6063) 2024-11-24T13:28:07,312 INFO [M:0;b10924491aa5:38765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2497726a6e4494bb4e1f9b86c2bf7cf 2024-11-24T13:28:07,332 DEBUG [M:0;b10924491aa5:38765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b1b425f67534125b41e96633d7189e4 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732454863891/Put/seqid=0 2024-11-24T13:28:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741946_1126 (size=7906) 2024-11-24T13:28:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741946_1126 (size=7906) 2024-11-24T13:28:07,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741946_1126 (size=7906) 2024-11-24T13:28:07,341 INFO [M:0;b10924491aa5:38765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b1b425f67534125b41e96633d7189e4 2024-11-24T13:28:07,346 INFO [M:0;b10924491aa5:38765 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6b1b425f67534125b41e96633d7189e4 2024-11-24T13:28:07,364 DEBUG [M:0;b10924491aa5:38765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0cdd158db6ac4825b0d22ce76ffe984d is 69, key is b10924491aa5,38813,1732454849031/rs:state/1732454850240/Put/seqid=0 2024-11-24T13:28:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741947_1127 (size=5440) 2024-11-24T13:28:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741947_1127 (size=5440) 2024-11-24T13:28:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741947_1127 (size=5440) 2024-11-24T13:28:07,371 INFO [M:0;b10924491aa5:38765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0cdd158db6ac4825b0d22ce76ffe984d 2024-11-24T13:28:07,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:28:07,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x1014a28febd0001, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:28:07,371 INFO [RS:0;b10924491aa5:38813 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:28:07,371 INFO [RS:0;b10924491aa5:38813 {}] regionserver.HRegionServer(1031): Exiting; stopping=b10924491aa5,38813,1732454849031; zookeeper connection closed. 2024-11-24T13:28:07,371 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@218249d4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@218249d4 2024-11-24T13:28:07,372 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T13:28:07,376 INFO [M:0;b10924491aa5:38765 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0cdd158db6ac4825b0d22ce76ffe984d 2024-11-24T13:28:07,377 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d2497726a6e4494bb4e1f9b86c2bf7cf as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d2497726a6e4494bb4e1f9b86c2bf7cf 2024-11-24T13:28:07,382 INFO [M:0;b10924491aa5:38765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d2497726a6e4494bb4e1f9b86c2bf7cf, entries=14, sequenceid=207, filesize=5.9 K 2024-11-24T13:28:07,383 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b1b425f67534125b41e96633d7189e4 as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6b1b425f67534125b41e96633d7189e4 2024-11-24T13:28:07,388 INFO [M:0;b10924491aa5:38765 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6b1b425f67534125b41e96633d7189e4 2024-11-24T13:28:07,388 INFO [M:0;b10924491aa5:38765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6b1b425f67534125b41e96633d7189e4, entries=21, sequenceid=207, filesize=7.7 K 2024-11-24T13:28:07,389 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0cdd158db6ac4825b0d22ce76ffe984d as hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0cdd158db6ac4825b0d22ce76ffe984d 2024-11-24T13:28:07,393 INFO [M:0;b10924491aa5:38765 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0cdd158db6ac4825b0d22ce76ffe984d 2024-11-24T13:28:07,394 INFO [M:0;b10924491aa5:38765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35337/user/jenkins/test-data/27ef28c9-7473-3b56-06d4-9ce249561b07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0cdd158db6ac4825b0d22ce76ffe984d, entries=3, sequenceid=207, filesize=5.3 K 2024-11-24T13:28:07,395 INFO [M:0;b10924491aa5:38765 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=207, compaction requested=false 2024-11-24T13:28:07,396 INFO [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:28:07,396 DEBUG [M:0;b10924491aa5:38765 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732454887289Disabling compacts and flushes for region at 1732454887289Disabling writes for close at 1732454887289Obtaining lock to block concurrent updates at 1732454887289Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732454887289Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1732454887289Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732454887290 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732454887290Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732454887304 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732454887304Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732454887316 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732454887331 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732454887332 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732454887346 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732454887362 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732454887363 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17be94e4: reopening flushed file at 1732454887376 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4681d72f: reopening flushed file at 1732454887382 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42759e1d: reopening flushed file at 1732454887388 (+6 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=207, compaction requested=false at 1732454887395 (+7 ms)Writing region close event to WAL at 1732454887396 (+1 ms)Closed at 1732454887396 2024-11-24T13:28:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46321 is added to blk_1073741830_1006 (size=99424) 2024-11-24T13:28:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741830_1006 (size=99424) 2024-11-24T13:28:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36111 is added to blk_1073741830_1006 (size=99424) 2024-11-24T13:28:07,399 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:28:07,399 INFO [M:0;b10924491aa5:38765 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T13:28:07,399 INFO [M:0;b10924491aa5:38765 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38765 2024-11-24T13:28:07,400 INFO [M:0;b10924491aa5:38765 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:28:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:28:07,502 INFO [M:0;b10924491aa5:38765 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:28:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38765-0x1014a28febd0000, quorum=127.0.0.1:49808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:28:07,508 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454877080 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454877080 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732454867843/wal.1732454877080 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T13:28:07,510 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867710 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867710 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732454867445/wal.1732454867710 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T13:28:07,511 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732454858929/wal.1732454859050 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:07,513 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877816 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877816 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732454877269/wal.1732454877816 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T13:28:07,514 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732454877959/wal.1732454878025 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:07,514 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732454882374/wal.1732454882563 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:07,514 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732454852562/wal.1732454852628 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:28:07,516 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal.1732454852436 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal.1732454852436 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732454851865/wal.1732454852436 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T13:28:07,518 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal.1732454851564 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal.1732454851564 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732454851309/wal.1732454851564 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1106905432_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T13:28:07,526 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T13:28:07,528 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T13:28:07,528 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T13:28:07,528 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T13:28:07,528 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,STOPPED} 2024-11-24T13:28:07,531 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T13:28:07,531 WARN [BP-1725618806-172.17.0.2-1732454845323 heartbeating to localhost/127.0.0.1:35337 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T13:28:07,531 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T13:28:07,531 WARN [BP-1725618806-172.17.0.2-1732454845323 heartbeating to localhost/127.0.0.1:35337 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1725618806-172.17.0.2-1732454845323 (Datanode Uuid 4f5fc3d0-59f3-4f3f-a741-6ccc0cb7aad3) service to localhost/127.0.0.1:35337 2024-11-24T13:28:07,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data5/current/BP-1725618806-172.17.0.2-1732454845323 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T13:28:07,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data6/current/BP-1725618806-172.17.0.2-1732454845323 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T13:28:07,534 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T13:28:07,535 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T13:28:07,536 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T13:28:07,536 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T13:28:07,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T13:28:07,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,STOPPED} 2024-11-24T13:28:07,537 WARN [BP-1725618806-172.17.0.2-1732454845323 heartbeating to localhost/127.0.0.1:35337 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T13:28:07,537 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T13:28:07,538 WARN [BP-1725618806-172.17.0.2-1732454845323 heartbeating to localhost/127.0.0.1:35337 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1725618806-172.17.0.2-1732454845323 (Datanode Uuid 5a7d5ab0-33c9-429d-ba0d-bfc5a68ff26d) service to localhost/127.0.0.1:35337 2024-11-24T13:28:07,538 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T13:28:07,538 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T13:28:07,539 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data3/current/BP-1725618806-172.17.0.2-1732454845323 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T13:28:07,539 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data4/current/BP-1725618806-172.17.0.2-1732454845323 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T13:28:07,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T13:28:07,541 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T13:28:07,541 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T13:28:07,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T13:28:07,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,STOPPED} 2024-11-24T13:28:07,542 WARN [BP-1725618806-172.17.0.2-1732454845323 heartbeating to localhost/127.0.0.1:35337 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T13:28:07,542 WARN [BP-1725618806-172.17.0.2-1732454845323 heartbeating to localhost/127.0.0.1:35337 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1725618806-172.17.0.2-1732454845323 (Datanode Uuid 5534b7c0-b2ba-4c2e-b79c-d95b1343c18b) service to localhost/127.0.0.1:35337 2024-11-24T13:28:07,542 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T13:28:07,542 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T13:28:07,543 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data1/current/BP-1725618806-172.17.0.2-1732454845323 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T13:28:07,543 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/cluster_0ba3dcb1-3103-2f27-942b-fda8d577063e/data/data2/current/BP-1725618806-172.17.0.2-1732454845323 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T13:28:07,543 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T13:28:07,550 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T13:28:07,550 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T13:28:07,550 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T13:28:07,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T13:28:07,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/131313f9-b376-ca2a-f347-78e61f5a3888/hadoop.log.dir/,STOPPED} 2024-11-24T13:28:07,560 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T13:28:07,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down